Merge branch 'wip-configurable-strategy' into 'master'

Configurable strategy

Implementing a strategy configurable via an external `config.js` file.

See merge request rysiek/samizdat!28

README.md

@@ -65,7 +65,7 @@ The data provided (per each requested URL handled by the ServiceWorker) is:
  - `url` – the URL of the request
  - `serviceWorker` – the commit SHA of the ServiceWorker that handled the request
  - `fetchError` – `null` if the request completed successfully via regular HTTPS; otherwise the error message
- - `method` – the method by which the request was completed: "`fetch`" is regular HTTPS `fetch()`, `gun-ipns` means Gun and IPFS were used, etc.
+ - `method` – the method by which the request was completed: "`fetch`" is regular HTTPS `fetch()`, `gun-ipfs` means Gun and IPFS were used, etc.
  - `state` – the state of the request (`running`, `error`, `success`)
 
 The code in the browser window context is responsible for keeping a more permanent record of the URLs requested, the methods used, and the status of each, if needed.

config.js

@@ -8,15 +8,17 @@
  */
 
 // plugins config
-self.SamizdatConfig.plugins["gateway-ipns"] = {
-    // the pubkey of the preconfigured IPNS node
-    ipnsPubkey: 'QmYGVgGGfD5N4Xcc78CcMJ99dKcH6K6myhd4Uenv5yJwiJ'
-}
-self.SamizdatConfig.plugins["gun-ipns"] = {
-    // the pubkey of the preconfigured Gun user
-    gunPubkey: 'WUK5ylwqqgUorceQRa84qfbBFhk7eNRDUoPbGK05SyE.-yohFhTzWPpDT-UDMuKGgemOUrw_cMMYWpy6plILqrg'
-}
-self.SamizdatConfig.plugins["ipns+ipfs"] = {
-    // the pubkey of the preconfigured Gun user
-    ipnsPubkey: 'QmYGVgGGfD5N4Xcc78CcMJ99dKcH6K6myhd4Uenv5yJwiJ'
-}
+self.SamizdatConfig.plugins = {
+                        'fetch':{},
+                        'cache':{},
+                        'any-of': {
+                            plugins: {
+                                'gateway-ipns':{
+                                    ipnsPubkey: 'QmYGVgGGfD5N4Xcc78CcMJ99dKcH6K6myhd4Uenv5yJwiJ'
+                                },
+                                'gun-ipfs': {
+                                    gunPubkey: 'WUK5ylwqqgUorceQRa84qfbBFhk7eNRDUoPbGK05SyE.-yohFhTzWPpDT-UDMuKGgemOUrw_cMMYWpy6plILqrg'
+                                }
+                            }
+                        }
+                    }

docs/ARCHITECTURE.md

@@ -51,7 +51,7 @@ Whenever a resource is being fetched on a Samizdat-enabled site, the `service-wo
 
 1. `fetch`, to use the upstream site directly if it is available,
 1. `cache`, to display the site immediately from the cache in case regular `fetch` fails,
-1. `gun-ipns`, in the background if `cache` call succeeded, otherwise as the active fetch handler.
+1. `gun-ipfs`, in the background if `cache` call succeeded, otherwise as the active fetch handler.
 
 If a background plugin `fetch()` succeeds, the result is added to the cache and will be immediately available on page reload.
 

docs/DEPLOYMENT.md

@@ -5,7 +5,7 @@
 Samizdat deployment is currently mostly a manual process. It involves:
 1. getting the code
 1. creating a [Gun](https://gun.eco/) account to push content addresses to
-1. replacing the Gun account public key in the `gun-ipns` plugin code
+1. replacing the Gun account public key in the `gun-ipfs` plugin code
 1. deploying the code to your server
 
 This is highly suboptimal; work is being done to design and implement a process that is more user-friendly and automated. In the meantime, you can deploy Samizdat by following these steps.

docs/OVERVIEW.md

@@ -149,7 +149,7 @@ When a request is made, plugins are used to handle it in the order defined in co
 1. retrieval from local `cache`;
 1. any other transport plugins (`IPFS`, or fetching content from other pre-configured endpoints).
 
-It is up to the website admin's to configure the plugins in a way that makes sense. For example, if using the [`gun-ipns`](./../plugins/gun-ipfs.js) plugin, the admin needs to create a Gun account and populate the plugin's Gun public key variable. If using `IPNS`, the admin needs to populate the `IPNS` public key in the respective plugin. And if alternative HTTPS endpoints are used, it's up to the admin to populate their URLs.
+It is up to the website admin's to configure the plugins in a way that makes sense. For example, if using the [`gun-ipfs`](./../plugins/gun-ipfs.js) plugin, the admin needs to create a Gun account and populate the plugin's Gun public key variable. If using `IPNS`, the admin needs to populate the `IPNS` public key in the respective plugin. And if alternative HTTPS endpoints are used, it's up to the admin to populate their URLs.
 
 Examples of possible alternative HTTPS locations:
  - an IP address or a domain controlled by the admin that is not linked to the original website and thus can be expected not to be blocked;
@@ -161,7 +161,7 @@ Examples of possible alternative HTTPS locations:
 
 For content to be available for retrieval by transport plugins, it first needs to be published to whatever locations it is going to be retrieved from.
 
-For the `gun-ipns` plugin, for instance, content needs to first be published in IPFS, and then the IPFS addresses for each file need to be published to Gun. Currently it is an involved process (example is available in the [CI/CD configuration for the project](./../.gitlab-ci.yml)); making it simpler and easier is the focus of current development.
+For the `gun-ipfs` plugin, for instance, content needs to first be published in IPFS, and then the IPFS addresses for each file need to be published to Gun. Currently it is an involved process (example is available in the [CI/CD configuration for the project](./../.gitlab-ci.yml)); making it simpler and easier is the focus of current development.
 
 For plugins relying on fetching content from alternative HTTPS locations, this can be as simple as deploying the content to the alternative IP address or domain name, pushing the content to WebArchive, putting the content in the Google Drive folder, or uploading it to the CloudFront location.
 

plugins/any-of.js

+/* ========================================================================= *\
+|* === Any-of: running multiple plugins simultaneously                   === *|
+\* ========================================================================= */
+
+/**
+ * this plugin does not implement any push method
+ */
+
+// no polluting of the global namespace please
+(function () {
+    
+    /*
+     * plugin config settings
+     */
+    
+    // sane defaults
+    let defaultConfig = {
+        // list of plugins to run simultaneously
+        plugins: {
+            "gateway-ipns": {},
+            "gun-ipfs": {}
+        }
+    }
+    
+    // merge the defaults with settings from SamizdatConfig
+    let config = {...defaultConfig, ...self.SamizdatConfig.plugins["any-of"]}
+    
+    /**
+     * getting content using regular HTTP(S) fetch()
+     */
+    let fetchContent = (url) => {
+        console.log(`Samizdat: any-of: [${Object.keys(config.plugins).join(', ')}]!`)
+        return Promise.any(
+            SamizdatPlugins
+                .filter(p=>Object.keys(config.plugins).includes(p.name))
+                .map(p=>p.fetch(url))
+        )
+    }
+
+    // and add ourselves to it
+    // with some additional metadata
+    self.SamizdatPlugins.push({
+        name: 'any-of',
+        description: `Running simultaneously: [${Object.keys(config.plugins).join(', ')}]`,
+        version: 'COMMIT_UNKNOWN',
+        fetch: fetchContent,
+        uses: config.plugins
+    })
+
+// done with not poluting the global namespace
+})()

plugins/cache.js

@@ -119,11 +119,6 @@
             })
     }
 
-    
-    // initialize the SamizdatPlugins array
-    if (!Array.isArray(self.SamizdatPlugins)) {
-        self.SamizdatPlugins = new Array()
-    }
 
     // and add ourselves to it
     // with some additional metadata

plugins/fetch.js

@@ -55,11 +55,6 @@
                         })
             })
     }
-    
-    // initialize the SamizdatPlugins array
-    if (!Array.isArray(self.SamizdatPlugins)) {
-        self.SamizdatPlugins = new Array()
-    }
 
     // and add ourselves to it
     // with some additional metadata

plugins/gateway-ipns.js

@@ -4,6 +4,10 @@
 
 /**
  * this plugin does not implement any push method
+ * 
+ * NOTICE: this plugin uses Promise.any()
+ *         https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/any
+ *         the polyfill is implemented in Samizdat's service-worker.js
  */
 
 // no polluting of the global namespace please
@@ -40,30 +44,7 @@
         console.error(err)
         throw err
     }
-
-    /*
-     * to do this right we need a Promise.any() polyfill
-     * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/any
-     */
-    Promise.any = async (promises) => {
-        // Promise.all() is the polar opposite of Promise.any()
-        // in that it returns as soon as there is a first rejection
-        // but without it, it returns an array of resolved results
-        return Promise.all(
-            promises.map(p => {
-                return new Promise((resolve, reject) =>
-                    // swap reject and resolve, so that we can use Promise.all()
-                    // and get the result we need
-                    Promise.resolve(p).then(reject, resolve)
-                );
-            })
-        // now, swap errors and values back
-        ).then(
-            err => Promise.reject(err),
-            val => Promise.resolve(val)
-        );
-    };
-
+    
 
     /**
      * getting content using regular HTTP(S) fetch()
@@ -134,11 +115,6 @@
                         })
             })
     }
-    
-    // initialize the SamizdatPlugins array
-    if (!Array.isArray(self.SamizdatPlugins)) {
-        self.SamizdatPlugins = new Array()
-    }
 
     // and add ourselves to it
     // with some additional metadata

plugins/gun-ipfs.js

@@ -27,7 +27,7 @@
     }
 
     // merge the defaults with settings from SamizdatConfig
-    let config = {...defaultConfig, ...self.SamizdatConfig.plugins["gun-ipns"]}
+    let config = {...defaultConfig, ...self.SamizdatConfig.plugins["gun-ipfs"]}
 
     // reality check: Gun pubkey needs to be set to a non-empty string
     if (typeof(config.gunPubkey) !== "string" || config.gunPubkey === "") {
@@ -211,7 +211,7 @@
                             'headers': {
                                 'Content-Type': contentType,
                                 'ETag': file.value.path,
-                                'X-Samizdat-Method': 'gun-ipns',
+                                'X-Samizdat-Method': 'gun-ipfs',
                                 'X-Samizdat-ETag': file.value.path
                             }
                         }
@@ -441,15 +441,10 @@
     // we probably need to handle this better
     setup_gun_ipfs();
 
-    // initialize the SamizdatPlugins array
-    if (!Array.isArray(self.SamizdatPlugins)) {
-        self.SamizdatPlugins = new Array()
-    }
-
     // and add ourselves to it
     // with some additional metadata
     self.SamizdatPlugins.push({
-        name: 'gun-ipns',
+        name: 'gun-ipfs',
         description: 'Decentralized resource fetching using Gun for address resolution and IPFS for content delivery.',
         version: 'COMMIT_UNKNOWN',
         fetch: getContentFromGunAndIPFS,

plugins/ipns-ipfs.js

@@ -32,7 +32,7 @@
     }
 
     // merge the defaults with settings from SamizdatConfig
-    let config = {...defaultConfig, ...self.SamizdatConfig.plugins["ipns+ipfs"]}
+    let config = {...defaultConfig, ...self.SamizdatConfig.plugins["ipns-ipfs"]}
 
     // reality check: Gun pubkey needs to be set to a non-empty string
     if (typeof(config.ipnsPubkey) !== "string" || config.ipnsPubkey === "") {
@@ -168,7 +168,7 @@
                             'headers': {
                                 'Content-Type': contentType,
                                 'ETag': file.value.path,
-                                'X-Samizdat-Method': 'gun-ipns',
+                                'X-Samizdat-Method': 'ipns-ipfs',
                                 'X-Samizdat-ETag': file.value.path
                             }
                         }
@@ -324,15 +324,11 @@
     // we probably need to handle this better
     setup_ipfs();
 
-    // initialize the SamizdatPlugins array
-    if (!Array.isArray(self.SamizdatPlugins)) {
-        self.SamizdatPlugins = new Array()
-    }
 
     // and add ourselves to it
     // with some additional metadata
     self.SamizdatPlugins.push({
-        name: 'ipns+ipfs',
+        name: 'ipns-ipfs',
         description: 'Decentralized resource fetching using IPNS for address resolution and IPFS for content delivery.',
         version: 'COMMIT_UNKNOWN',
         fetch: getContentFromIPNSAndIPFS,

service-worker.js

@@ -9,6 +9,35 @@
  *       Samizdat.
  */
 
+
+/*
+ * we need a Promise.any() polyfill
+ * so here it is
+ * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/any
+ * 
+ * TODO: remove once Promise.any() is implemented broadly
+ */
+if (typeof Promise.any === 'undefined') {
+    Promise.any = async (promises) => {
+        // Promise.all() is the polar opposite of Promise.any()
+        // in that it returns as soon as there is a first rejection
+        // but without it, it returns an array of resolved results
+        return Promise.all(
+            promises.map(p => {
+                return new Promise((resolve, reject) =>
+                    // swap reject and resolve, so that we can use Promise.all()
+                    // and get the result we need
+                    Promise.resolve(p).then(reject, resolve)
+                );
+            })
+        // now, swap errors and values back
+        ).then(
+            err => Promise.reject(err),
+            val => Promise.resolve(val)
+        );
+    };
+}
+
 // initialize the SamizdatPlugins array
 if (!Array.isArray(self.SamizdatPlugins)) {
     self.SamizdatPlugins = new Array()
@@ -24,31 +53,71 @@ if (typeof self.SamizdatConfig !== 'object' || self.SamizdatConfig === null) {
         // and move on?
         defaultPluginTimeout: 10000,
         // plugins settings namespace
-        plugins: {},
+        //
         // this defines which plugins get loaded,
         // and the order in which they are deployed to try to retrieve content
         // assumption: plugin path = ./plugins/<plugin-name>.js
-        strategy: [
-            'fetch',
-            'cache',
-            'gateway-ipns',
-            'gun-ipfs'
-        ]
+        // 
+        // this relies on JavaScript preserving the insertion order for properties
+        // https://stackoverflow.com/a/5525820
+        plugins: {
+            'fetch':{},
+            'cache':{},
+            'gateway-ipns':{},
+            'gun-ipfs':{}
+        }
     }
 }
 
 // load the plugins
-//
-// order in which plugins are loaded defines the order
-// in which they are called!
+// 
+// everything in a try-catch block
+// so that we get an informative message if there's an error
 try {
+    
     // get the config
     self.importScripts("./config.js")
+    
     // only now load the plugins (config.js could have changed the defaults)
-    self.importScripts(
-        ...self.SamizdatConfig.strategy.map(
-            pluginName => `./plugins/${pluginName}.js`
-        ))
+    var plugins = Object.keys(self.SamizdatConfig.plugins)
+    for (var i=0; i<plugins.length; i++) {
+        // load a plugin
+        self.importScripts(`./plugins/${plugins[i]}.js`)
+        // check if it loaded properly
+        var plugin = SamizdatPlugins.find(p=>p.name===plugins[i])
+        if (plugin === undefined) {
+            throw new Error(`Plugin not found: ${plugins[i]} (available plugins: ${SamizdatPlugins.map(p=>p.name).join(', ')})`)
+        }
+        // make sure that the indirect flag is set if needed
+        if (self.SamizdatConfig.plugins[plugin.name].indirect===true) {
+            plugin.indirect=true
+            console.log(`Loaded plugin: ${plugin.name} (indirect)`)
+        } else {
+            console.log(`Loaded plugin: ${plugin.name}`)
+        }
+        // make sure plugins used by the just-loaded plugin are also added to the list
+        // but don't load a plugin twice
+        if (typeof plugin.uses !== "undefined") {
+            for (p in plugin.uses) {
+                if (plugins.indexOf(p) < 0) {
+                    // okay, this plugin has not been added to the plugins list yet
+                    // let's do that
+                    plugins.push(p)
+                    // but also, let's make sure that the config for them is available for use
+                    var pConfig = {...self.SamizdatConfig.plugins[p], ...plugin.uses[p]}
+                    // set the indirect flag,
+                    // since we only have this plugin here to facilitate use by other plugins
+                    pConfig.indirect = true
+                    // set the config
+                    self.SamizdatConfig.plugins[p] = pConfig
+                }
+            }
+        }
+    }
+    
+    // inform
+    console.log(`DEBUG: Strategy in use: ${SamizdatPlugins.filter(p=>(!p.indirect)).map(p=>p.name).join(', ')}`)
+
 } catch(e) {
     // we only get a cryptic "Error while registering a service worker"
     // unless we explicitly print the errors out in the console
@@ -97,7 +166,7 @@ let decrementActiveFetches = (clientId) => {
  * timeout_resolves - whether the Promise should resolve() or reject() when hitting the timeout (default: false (reject))
  * error_message    - optional error message to use when rejecting (default: false (no error message))
  */
-function promiseTimeout(time, timeout_resolves=false, error_message=false) {
+let promiseTimeout = (time, timeout_resolves=false, error_message=false) => {
     return new Promise((resolve, reject)=>{
         setTimeout(()=>{
             if (timeout_resolves) {
@@ -240,40 +309,27 @@ let SamizdatResourceInfo = class {
 |* === Main Brain of Samizdat                                            === *|
 \* ========================================================================= */
 
-/**
- * get a plugin by name
- * 
- * this doesn't have to be super-performant, since we should never have more
- * then a few plugins
- * (let's see how long it takes for me to eat my own words here)
- */
-let getSamizdatPluginByName = (name) => {
-    for (i=0; i<SamizdatPlugins.length; i++) {
-        if (SamizdatPlugins[i].name === name) {
-            return SamizdatPlugins[i]
-        }
-    }
-    return null
-}
-
 /**
  * run a plugin's fetch() method
  * while handling all the auxiliary stuff like saving info in reqInfo
  * 
- * plugin     - the plugin to use
- * url        - string containing the URL to fetch
- * lastError  - error thrown by the previous plugin, if any (default: null)
+ * plugin    - the plugin to use
+ * url       - string containing the URL to fetch
+ * reqInfo   - instance of SamizdatResourceInfo
  */
 let samizdatFetch = (plugin, url, reqInfo) => {
-    // status of the current method
+    
+    // status of the plugin
     reqInfo.update({
         method: plugin.name,
         state: "running"
     })
+    
     // log stuff
-    console.log("(COMMIT_UNKNOWN) Samizdat handling URL:", url,
-                '\n+-- current method : ' + plugin.name)
-    // run the plugin
+    console.log("(COMMIT_UNKNOWN) Samizdat Service Worker handling URL:", url,
+                '\n+-- using method(s):', plugin.name)
+    
+    // race the plugin(s) vs. a timeout
     return Promise.race([
         plugin.fetch(url),
         promiseTimeout(
@@ -286,13 +342,13 @@ let samizdatFetch = (plugin, url, reqInfo) => {
 
 
 /**
- * calling a samizdat plugin function
+ * calling a samizdat plugin function on the first plugin that implements it
  * 
  * call - method name to call
  * args - arguments that will be passed to it
  */
 let callOnSamizdatPlugin = (call, args) => {
-    // find the first method implementing the method
+    // find the first plugin implementing the method
     for (i=0; i<SamizdatPlugins.length; i++) {
         if (typeof SamizdatPlugins[i][call] === 'function') {
             console.log('(COMMIT_UNKNOWN) Calling plugin ' + SamizdatPlugins[i].name + '.' + call + '()')
@@ -326,7 +382,7 @@ let getResourceThroughSamizdat = (request, clientId, useStashed=true, doStash=tr
     
     // filter out stash plugins if need be
     var SamizdatPluginsRun = SamizdatPlugins.filter((plugin)=>{
-        return (useStashed || typeof plugin.stash !== 'function')
+        return ( (!plugin.indirect) && (useStashed || typeof plugin.stash !== 'function') )
     })
     
     /**
@@ -374,7 +430,7 @@ let getResourceThroughSamizdat = (request, clientId, useStashed=true, doStash=tr
         reqInfo.update({state:"success"})
         
         // get the plugin that was used to fetch content
-        plugin = getSamizdatPluginByName(reqInfo.method)
+        plugin = SamizdatPlugins.find(p=>p.name===reqInfo.method)
         
         // if it's a stashing plugin...
         if (typeof plugin.stash === 'function') {
@@ -432,13 +488,6 @@ let getResourceThroughSamizdat = (request, clientId, useStashed=true, doStash=tr
                         response.headers.forEach(function(v, k){
                             console.log('+-- Stashing header: ', k, ' :: ', v)
                         });
-
-                        // var cacheRequest = new Request(request, {
-                        //     headers: new Headers({
-                        //         'X-Samizdat-Method': reqInfo.method,
-                        //         'X-Samizdat-Etag': response.headers['X-Samizdat-ETag']
-                        //     })
-                        // })
                         
                         // working on clone()'ed response so that the original one is not touched
                         // TODO: should a failed stashing break the flow here? probably not!
@@ -513,7 +562,7 @@ self.addEventListener('fetch', event => {
             .then((client)=>{
                 client.postMessage({
                     clientId: clientId,
-                    plugins: SamizdatPlugins.map((p)=>{return p.name}),
+                    plugins: SamizdatPlugins.filter(p=>(!p.indirect)).map((p)=>{return p.name}),
                     serviceWorker: 'COMMIT_UNKNOWN'
                 })
             })