gulpfile.js

"use strict";

// dependencies
var path          = require("path");
var fs            = require("fs");
var fse           = require("fs-extra");
var child_process = require("child_process");

var gulp        = require("gulp");
var gutil       = require("gulp-util");
var less        = require("gulp-less");
var sass        = require("gulp-sass");
var replace     = require("gulp-replace");
var header      = require("gulp-header");
var footer      = require("gulp-footer");
var rename      = require("gulp-rename");
var browsersync = require("browser-sync");
var vstream     = require("vinyl-source-stream");
var buffer      = require("vinyl-buffer");

var browserify = require("browserify");
var babelify   = require("babelify");
var uglify     = require("gulp-uglify");
var envify     = require("envify");
var htmllint   = require("gulp-htmllint");
var crawler    = require("simplecrawler");
var ncp        = require("ncp");

var nextversion = require("./tools/bin/nextversion");
var util        = require("./tools/bin/util");

// constants
var ROOT_DIR   = ".";
var CONFIG_DIR = "conf";
var SOURCE_DIR = path.join(ROOT_DIR, "www");
var DEV_DIR    = path.join(ROOT_DIR, "build-dev");
var PROD_DIR   = path.join(ROOT_DIR, "build-prod");

var DATA_DIR        = path.join(SOURCE_DIR, "_data");
var TOC_DIR         = path.join(DATA_DIR, "toc");
var DOCS_DIR        = path.join(SOURCE_DIR, "docs");
var FETCH_DIR       = path.join(DOCS_DIR, "en", "dev", "reference");
var CSS_SRC_DIR     = path.join(SOURCE_DIR, "static", "css-src");
var CSS_OUT_DIR     = path.join(SOURCE_DIR, "static", "css");
var PLUGINS_SRC_DIR = path.join(SOURCE_DIR, "static", "plugins");
var JS_DIR          = path.join(SOURCE_DIR, "static", "js");
var BIN_DIR         = path.join(ROOT_DIR, "tools", "bin");

var CONFIG_FILE          = path.join(CONFIG_DIR, "_config.yml");
var DEFAULTS_CONFIG_FILE = path.join(CONFIG_DIR, "_defaults.yml");
var VERSION_CONFIG_FILE  = path.join(CONFIG_DIR, "_version.yml");
var PROD_CONFIG_FILE     = path.join(CONFIG_DIR, "_prod.yml");
var DEV_CONFIG_FILE      = path.join(CONFIG_DIR, "_dev.yml");
var NODOCS_CONFIG_FILE   = path.join(CONFIG_DIR, "_nodocs.yml");

var VERSION_FILE      = "VERSION";
var DOCS_VERSION_FILE = path.join(DATA_DIR, "docs-versions.yml");
var ALL_PAGES_FILE    = path.join(DATA_DIR, "all-pages.yml");
var FETCH_CONFIG      = path.join(DATA_DIR, "fetched-files.yml");
var REDIRECTS_FILE    = path.join(DATA_DIR, "redirects.yml");
var PLUGINS_FILE_NAME = "plugins.js";
var PLUGINS_FILE      = path.join(JS_DIR, PLUGINS_FILE_NAME);
var PLUGINS_SRC_FILE  = path.join(PLUGINS_SRC_DIR, "app.js");

var BASE_CONFIGS = [CONFIG_FILE, DEFAULTS_CONFIG_FILE, VERSION_CONFIG_FILE];
var DEV_CONFIGS  = [DEV_CONFIG_FILE];
var PROD_CONFIGS = [PROD_CONFIG_FILE];
var DEV_FLAGS    = ["--trace"];
var PROD_FLAGS   = [];

var BASE_URL            = "";
var YAML_FRONT_MATTER   = "---\n---\n";
var WATCH_INTERVAL      = 1000; // in milliseconds
var VERSION_VAR_NAME    = "latest_docs_version";
var LATEST_DOCS_VERSION = fs.readFileSync(VERSION_FILE, 'utf-8').trim();
var NEXT_DOCS_VERSION   = nextversion.getNextVersion(LATEST_DOCS_VERSION);
var LANGUAGES           = util.listdirsSync(DOCS_DIR);

var PROD_BY_DEFAULT = false;

// compute/get/set/adjust passed options
gutil.env.prod   = gutil.env.prod || PROD_BY_DEFAULT;
gutil.env.dev    = !gutil.env.prod;
gutil.env.outDir = gutil.env.prod ? PROD_DIR : DEV_DIR;

// check for errors
if (gutil.env.prod && gutil.env.nodocs) {
    fatal("can't ignore docs when doing a production build");
}

// helpers
function fatal(message) {
    gutil.log(gutil.colors.red("ERROR") + ": " + message);
    process.exit(1);
}

function execPiped(command, args, fileName) {
    console.log(command + " " + args.join(" "));
    var task = child_process.spawn(command, args);
    return task.stdout.pipe(vstream(fileName)).pipe(buffer());
}

function exec(command, args, cb) {
    console.log(command + " " + args.join(" "));
    var task = child_process.spawn(command, args, {stdio: "inherit"});
    task.on("exit", cb);
}

function bin(name) {
    return path.join(BIN_DIR, name);
}

function remove(path) {
    console.log("removing " + path);
    fse.removeSync(path);
}

function getBundleExecutable() {
    if (process.platform === "win32") {
        return "bundle.bat";
    } else {
        return "bundle";
    }
}

function getJekyllConfigs() {
    var configs = BASE_CONFIGS;

    // add build-specific config files
    if (gutil.env.prod) {
        configs = configs.concat(PROD_CONFIGS);
    } else {
        configs = configs.concat(DEV_CONFIGS);
    }

    // add a special exclude file if "nodocs" was specified
    if (gutil.env.nodocs) {
        configs = configs.concat(NODOCS_CONFIG_FILE);
    }

    return configs;
}

function jekyllBuild(done) {
    var bundle  = getBundleExecutable();
    var configs = getJekyllConfigs();
    var flags   = gutil.env.prod ? PROD_FLAGS : DEV_FLAGS;

    flags = flags.concat(["--config", configs.join(",")]);

    exec(bundle, ["exec", "jekyll", "build"].concat(flags), done);
}

function copyDocsVersion(oldVersion, newVersion, cb) {

    // copying a folder and a ToC file for each language
    var numCopyOperations = LANGUAGES.length * 2;

    // pseudo-CV (condition variable)
    var numCopied = 0;
    function doneCopying(error) {

        if (error) {
            cb(error);
            return;
        }

        // call callback if all folders have finished copying
        numCopied += 1;
        if (numCopied === numCopyOperations) {
            cb();
        }
    }

    // create a new version for each language
    LANGUAGES.forEach(function (languageName) {

        // get files to copy
        var oldVersionDocs = path.join(DOCS_DIR, languageName, oldVersion);
        var oldVersionToc  = path.join(TOC_DIR, util.srcTocfileName(languageName, oldVersion));
        var newVersionDocs = path.join(DOCS_DIR, languageName, newVersion);
        var newVersionToc  = path.join(TOC_DIR, util.srcTocfileName(languageName, newVersion));

        var copyOptions = {
            stopOnErr: true
        };

        // copy docs
        console.log(oldVersionDocs + " -> " + newVersionDocs);
        ncp.ncp(oldVersionDocs, newVersionDocs, copyOptions, doneCopying);

        // copy ToC
        console.log(oldVersionToc + " -> " + newVersionToc);
        ncp.ncp(oldVersionToc, newVersionToc, copyOptions, doneCopying);
    });
}

// tasks
gulp.task("default", ["help"]);

gulp.task("help", function () {
    gutil.log("");
    gutil.log("Tasks:");
    gutil.log("");
    gutil.log("    build         same as configs + data + styles + plugins + jekyll");
    gutil.log("    jekyll        build with jekyll");
    gutil.log("    regen         same as jekyll + reload");
    gutil.log("    serve         build the site and open it in a browser");
    gutil.log("    reload        refresh the browser");
    gutil.log("");
    gutil.log("    newversion    create " + NEXT_DOCS_VERSION + " docs from dev docs");
    gutil.log("    snapshot      copy dev docs to " + LATEST_DOCS_VERSION + " docs");
    gutil.log("");
    gutil.log("    plugins       build " + PLUGINS_FILE);
    gutil.log("");
    gutil.log("    configs       run all the below tasks");
    gutil.log("    defaults      create " + DEFAULTS_CONFIG_FILE);
    gutil.log("    version       create " + VERSION_CONFIG_FILE);
    gutil.log("");
    gutil.log("    data          run all the below tasks");
    gutil.log("    docs-versions create " + DOCS_VERSION_FILE);
    gutil.log("    pages-dict    create " + ALL_PAGES_FILE);
    gutil.log("    toc           create all generated ToC files in " + TOC_DIR);
    gutil.log("    fetch         download docs specified in " + FETCH_CONFIG);
    gutil.log("");
    gutil.log("    styles        run all the below tasks");
    gutil.log("    less          compile all .less files");
    gutil.log("    sass          compile all .scss files");
    gutil.log("    css           copy over all .css files");
    gutil.log("");
    gutil.log("    watch         serve + then watch all source files and regenerate as necessary");
    gutil.log("    link-bugs     replace CB-XXXX references with nice links");
    gutil.log("");
    gutil.log("    help          show this text");
    gutil.log("    clean         remove all generated files and folders");
    gutil.log("");
    gutil.log("Arguments:");
    gutil.log("    --nodocs      don't generate docs");
    gutil.log("    --prod        build for production; without it, will build dev instead");
    gutil.log("");
});

gulp.task("data", ["toc", "docs-versions", "pages-dict"])
gulp.task("configs", ["defaults", "version"]);
gulp.task("styles", ["less", "css", "sass"]);

gulp.task("watch", ["serve"], function () {
    gulp.watch(
        [
            path.join(CSS_SRC_DIR, "**", "*"),
        ],
        {interval: WATCH_INTERVAL},
        ["styles"]
    );
    gulp.watch(
        [
            path.join(PLUGINS_SRC_DIR, "**", "*.js"),
            path.join(PLUGINS_SRC_DIR, "**", "*.jsx"),
            path.join(PLUGINS_SRC_DIR, "**", "*.json"),
        ],
        {interval: WATCH_INTERVAL},
        ["plugins"]
    );
    gulp.watch(
        [
            path.join(ROOT_DIR, "**", "*.yml"),
            path.join(JS_DIR, "**", "*.js"),
            path.join(CSS_OUT_DIR, "**", "*.css"),

            // NOTE:
            //      watch all non-docs HTML, and only docs/en/dev HTML because
            //      versions other than dev usually don't change much; this is
            //      an optimization
            path.join(SOURCE_DIR, "_layouts", "*.html"),
            path.join(SOURCE_DIR, "_includes", "*.html"),
            path.join(SOURCE_DIR, "**", "*.html") + "!" + path.join(DOCS_DIR, "**"),
            path.join(SOURCE_DIR, "**", "*.md") + "!" + path.join(DOCS_DIR, "**") ,
            path.join(DOCS_DIR, "en", "dev", "**", "*.md") ,
            path.join(DOCS_DIR, "en", "dev", "**", "*.html"),
        ],
        {interval: WATCH_INTERVAL},
        ["regen"]
    );
});

gulp.task("serve", ["build"], function () {
    var route = {};

    // set site root for browsersync
    if (gutil.env.prod) {
        route[BASE_URL] = gutil.env.outDir;
    }

    browsersync({
        notify: true,
        server: {
            baseDir: gutil.env.outDir,
            routes: route
        }
    });
});

gulp.task("build", ["configs", "data", "styles", "plugins"], function (done) {
    jekyllBuild(done);
});

gulp.task("jekyll", function (done) {
    jekyllBuild(done);
});

gulp.task("regen", ["jekyll"], function () {
    browsersync.reload();
});

gulp.task("fetch", function (done) {

    // skip fetching if --nofetch was passed
    if (gutil.env.nofetch) {
        gutil.log(gutil.colors.yellow(
            "Skipping fetching external docs."));
        done();
        return;
    }

    exec("node", [bin("fetch_docs.js"), "--config", FETCH_CONFIG, '--docsRoot', DOCS_DIR], done);
});

gulp.task("reload", function () {
    browsersync.reload();
});

gulp.task("docs-versions", function () {
    return execPiped("node", [bin("gen_versions.js"), DOCS_DIR], DOCS_VERSION_FILE)
        .pipe(gulp.dest(ROOT_DIR));
});

gulp.task("pages-dict", function () {
    var args = [
        bin("gen_pages_dict.js"),
        '--siteRoot', SOURCE_DIR,
        '--redirectsFile', REDIRECTS_FILE,
        '--latestVersion', LATEST_DOCS_VERSION,
        '--languages', LANGUAGES.join(","),
    ];

    return execPiped("node", args, ALL_PAGES_FILE).pipe(gulp.dest(ROOT_DIR));
});

gulp.task("version", function () {
    // this code is stupid; it's basically the line:
    //      cat VERSION | sed -e 's/^/VERSION_VAR_NAME: /' > _version.yml
    // however we're in Gulp, and we need to support Windows...
    // so we contort it into a monster
    return gulp
        .src(VERSION_FILE)
        .pipe(header(VERSION_VAR_NAME + ": "))
        .pipe(footer("\n"))
        .pipe(rename(VERSION_CONFIG_FILE))
        .pipe(gulp.dest("."));
});

gulp.task("defaults", function () {
    return execPiped("node", [bin("gen_defaults.js"), DOCS_DIR, LATEST_DOCS_VERSION], DEFAULTS_CONFIG_FILE)
        .pipe(gulp.dest(ROOT_DIR));
});

gulp.task("toc", ["fetch"], function (done) {
    exec("node", [bin("toc.js"), DOCS_DIR, TOC_DIR], done);
});

gulp.task("less", function () {
    return gulp
        .src(path.join(CSS_SRC_DIR, "**", "*.less"))
        .pipe(less())
        .pipe(header(YAML_FRONT_MATTER))
        .pipe(gulp.dest(CSS_OUT_DIR))
        .pipe(gulp.dest(CSS_OUT_DIR.replace(SOURCE_DIR, gutil.env.outDir)))
        .pipe(browsersync.reload({stream: true}));
})

gulp.task("css", function () {
    return gulp
        .src(path.join(CSS_SRC_DIR, "**", "*.css"))
        .pipe(header(YAML_FRONT_MATTER))
        .pipe(gulp.dest(CSS_OUT_DIR))
        .pipe(gulp.dest(CSS_OUT_DIR.replace(SOURCE_DIR, gutil.env.outDir)))
        .pipe(browsersync.reload({stream: true}));
})

gulp.task("sass", function() {
    return gulp
        .src(path.join(CSS_SRC_DIR, "**", "*.scss"))
        .pipe(sass().on("error", sass.logError))
        .pipe(header(YAML_FRONT_MATTER))
        .pipe(gulp.dest(CSS_OUT_DIR))
        .pipe(gulp.dest(CSS_OUT_DIR.replace(SOURCE_DIR, gutil.env.outDir)))
        .pipe(browsersync.reload({stream: true}));
});

gulp.task("plugins", function() {
    if (gutil.env.prod) {
        process.env.NODE_ENV = "production";
    }

    var stream = browserify(PLUGINS_SRC_FILE, {debug: !gutil.env.prod})
        .transform(babelify, {
            presets: ["react"],
            plugins: [
                ["transform-react-jsx", {"pragma": "h"}]
            ]
        })
        .transform(envify)
        .bundle()
        .on("error", gutil.log)
        .pipe(vstream(PLUGINS_FILE_NAME))
        .pipe(buffer());

    if (gutil.env.prod) {
        stream = stream
            .pipe(uglify())
            .on("error", gutil.log);
    }

    return stream
        .pipe(gulp.dest(JS_DIR.replace(SOURCE_DIR, gutil.env.outDir)))
        .pipe(browsersync.reload({stream: true}))

        // NOTE:
        //      adding YAML front matter after doing everything
        //      else so that uglify doesn't screw it up
        .pipe(header(YAML_FRONT_MATTER))

        // WORKAROUND:
        //           minified JS has some things that look like
        //           Liquid tags, so we replace them manually
        .pipe(replace("){{", "){ {"))
        .pipe(gulp.dest(JS_DIR));
});

// convenience tasks
gulp.task("link-bugs", function (done) {
    exec(bin("linkify-bugs.sh"), [path.join(SOURCE_DIR, "_posts")], done);
});

gulp.task("lint", function() {
    return gulp.src(path.join("./", "**", "*.html"))
        .pipe(htmllint());
});

gulp.task("newversion", ["fetch"], function(done) {

    copyDocsVersion("dev", NEXT_DOCS_VERSION, function (error) {

        if (error) {
            console.error(error);
            done();
            return;
        }

        // finally update the version file with the new version
        fs.writeFile(VERSION_FILE, NEXT_DOCS_VERSION + "\n", done);
    });
});

gulp.task("snapshot", ["fetch"], function(done) {

    // remove current version first
    LANGUAGES.forEach(function (languageName) {
        var languageLatestDocs = path.join(DOCS_DIR, languageName, LATEST_DOCS_VERSION);
        remove(languageLatestDocs);
    });

    copyDocsVersion("dev", LATEST_DOCS_VERSION, done);
});

gulp.task("checklinks", function(done) {
    crawler
        .crawl("http://localhost:3000/")
        .on("fetch404", function(queueItem, response) {
            gutil.log(
                "Resource not found linked from " +
                queueItem.referrer + " to", queueItem.url
            );
            gutil.log("Status code: " + response.statusCode);
        })
        .on("complete", function(queueItem) {
            done();
        });
});

gulp.task("clean", function () {
    remove(DEV_DIR);
    remove(PROD_DIR);
    remove(FETCH_DIR);
    remove(path.join(DATA_DIR, "toc", "*-gen.yml"));
    remove(CSS_OUT_DIR);
    remove(PLUGINS_FILE);
    remove(DOCS_VERSION_FILE);
    remove(ALL_PAGES_FILE);
    remove(DEFAULTS_CONFIG_FILE);
    remove(VERSION_CONFIG_FILE);
});