Only download cache for system languages and English (#420)

Co-authored-by: Matthew Peveler <[email protected]>

Author: vivekjoshi556

config.json

@@ -1,6 +1,6 @@
 {
   "pagesRepository": "https://github.com/tldr-pages/tldr",
-  "repository": "https://tldr.sh/assets/tldr.zip",
+  "repositoryBase": "https://tldr.sh/assets/tldr-pages",
   "skipUpdateWhenPageNotFound": false,
   "themes": {
     "simple": {

lib/cache.js

@@ -48,11 +48,13 @@ class Cache {
     return Promise.all([
       // Create new temporary folder
       fs.ensureDir(tempFolder),
-      fs.ensureDir(this.cacheFolder)
+      fs.ensureDir(this.cacheFolder),
     ])
       .then(() => {
         // Download and extract cache data to temporary folder
-        return remote.download(tempFolder);
+        return Promise.allSettled(this.config.languages.map((lang) => {
+          return remote.download(tempFolder, lang);
+        }));
       })
       .then(() => {
         // Copy data to cache folder
@@ -62,7 +64,7 @@ class Cache {
         return Promise.all([
           // Remove temporary folder
           fs.remove(tempFolder),
-          index.rebuildPagesIndex()
+          index.rebuildPagesIndex(),
         ]);
       })
       // eslint-disable-next-line no-unused-vars

lib/config.js

@@ -3,6 +3,7 @@
 const defaults = require('lodash/defaults');
 const fs = require('fs');
 const path = require('path');
+const utils = require('./utils');
 const osHomedir = require('os').homedir;
 
 exports.get = () => {
@@ -35,6 +36,22 @@ exports.get = () => {
   if (errors.length > 0) {
     throw new Error('Error in .tldrrc configuration:\n' + errors.join('\n'));
   }
+
+  // Setting correct languages
+  merged.languages = ['en'];
+  // Get the primary & secondary language.
+  let langs = utils.localeToLang(process.env.LANG);
+  merged.languages = merged.languages.concat(langs);
+  
+  if(process.env.LANGUAGE !== undefined) {
+    let langs = process.env.LANGUAGE.split(':');
+    
+    merged.languages.push(...langs.map((lang) => {
+      return utils.localeToLang(lang);
+    }));
+  }
+  merged.languages = [...new Set(merged.languages)];
+  
   return merged;
 };
 

lib/remote.js

@@ -1,34 +1,44 @@
 'use strict';
 
-const unzip = require('node-unzip-2');
+const path = require('path');
+const fs = require('fs-extra');
+const unzip = require('adm-zip');
 const config = require('./config');
 const axios = require('axios');
 
 // Downloads the zip file from github and extracts it to folder
-exports.download = (path) => {
-  const url = config.get().repository;
+exports.download = (loc, lang) => {
+  // If the lang is english then keep the url simple, otherwise add language.
+  const suffix = (lang === 'en' ? '' : '.' + lang);
+  const url = config.get().repositoryBase + suffix + '.zip';
+  const folderName = path.join(loc, 'pages' + suffix);
+  const REQUEST_TIMEOUT = 10000;
 
-  // Creating the extractor
-  const extractor = unzip.Extract({ path });
-
-  let req = axios({
+  return axios({
     method: 'get',
     url: url,
     responseType: 'stream',
-    headers: { 'User-Agent' : 'tldr-node-client' }
-  }).then(function (response) {
-    response.data.pipe(extractor);
-  });
+    headers: { 'User-Agent' : 'tldr-node-client' },
+    timeout: REQUEST_TIMEOUT,
+  }).then((response) => {
+    return new Promise((resolve, reject) => {
+      let fileName = path.join(loc, 'download_' + lang + '.zip');
 
-  return new Promise((resolve, reject) => {
-    req.catch((err) => {
-      reject(err);
-    });
-    extractor.on('error', () => {
-      reject(new Error('Cannot update from ' + url));
-    });
-    extractor.on('close', () => {
-      resolve();
+      const writer = fs.createWriteStream(fileName);
+      response.data.pipe(writer);
+    
+      writer.on('finish', () => {
+        writer.end();
+        const zip = new unzip(fileName);
+
+        zip.extractAllTo(folderName, true);
+        fs.unlinkSync(fileName);
+        resolve();
+      }).on('error', (err) => {
+        reject(err);
+      });
     });
+  }).catch((err) => {
+    return Promise.reject(err);
   });
-};
+};

lib/utils.js

@@ -24,6 +24,31 @@ module.exports = {
     return langParts[1];
   },
 
+  localeToLang(locale) {
+    if(locale === undefined || locale.startsWith('en')) return [];
+
+    const withDialect = ['pt', 'zh'];
+    
+    let lang = locale;
+    if(lang.includes('.')) {
+      lang = lang.substring(0, lang.indexOf('.'));
+    }
+    
+    // Check for language code & country code.
+    let ll = lang, cc = '';
+    if(lang.includes('_')) {
+      cc = lang.substring(lang.indexOf('_') + 1);
+      ll = lang.substring(0, lang.indexOf('_'));
+    }
+
+    // If we have dialect for this language take dialect as well.
+    if(withDialect.indexOf(ll) !== -1 && cc !== '') {
+      return [ll, ll + '_' + cc];
+    }
+
+    return [ll];
+  },
+
   isPage(file) {
     return path.extname(file) === '.md';
   },