Browse Source

New script-building pipeline

pull/60/merge
Lal'C Mellk Mal 7 years ago
parent
commit
89df58a18b
12 changed files with 255 additions and 117 deletions
  1. +4
    -0
      .gitignore
  2. +6
    -18
      Makefile
  3. +1
    -1
      README.md
  4. +1
    -16
      builder.js
  5. +6
    -2
      deps.js
  6. +11
    -0
      etc.js
  7. +0
    -36
      get.js
  8. +13
    -18
      make_client.js
  9. +163
    -0
      pipeline.js
  10. +9
    -3
      server/server.js
  11. +40
    -23
      server/state.js
  12. +1
    -0
      tmpl/index.html

+ 4
- 0
.gitignore View File

@@ -1,8 +1,10 @@
*.node
*.pid
*.swp
.build
.lock*
.DS_Store
assets.json
authdump
config.js
graveyard
@@ -12,12 +14,14 @@ jsmin
node_modules/
perceptual
imager/tmp
state
tripcode/.build
tripcode/build
upkeep/credentials.json
voice/*.mp3
www/archive
www/js/client*.js
www/js/vendor*.js
www/src
www/thumb
www/mid


+ 6
- 18
Makefile View File

@@ -1,26 +1,14 @@
CLIENT_SRC := $(shell node get.js CLIENT_DEPS)
DEBUG := $(shell node get.js DEBUG)
ifeq ($(DEBUG),true)
CLIENT_JS := www/js/client.debug.js
else
CLIENT_JS := www/js/client-$(shell node get.js --client-version).js
endif

all: client
all:
$(MAKE) -C imager
$(MAKE) -C tripcode

$(CLIENT_JS): $(CLIENT_SRC) config.js deps.js
node make_client.js $(CLIENT_SRC) > $@

client: $(CLIENT_JS)

modjs:
node make_client.js $(shell node get.js MOD_CLIENT_DEPS)
client:
@echo 'make client' is no longer necessary
@false

.PHONY: all client clean modjs
.PHONY: all clean client

clean:
rm -rf -- .build www/js/client{.,-}*.js
rm -rf -- .build state www/js/{client,vendor}{.,-}*.js
$(MAKE) -C imager -w clean
$(MAKE) -C tripcode -w clean

+ 1
- 1
README.md View File

@@ -15,7 +15,7 @@ Production:
* Have your webserver serve www/ (or wherever you've moved src, thumb, etc.)
* Run `node server/server.js` for just the server
* config.DAEMON support is broken for now
* Be sure to `make client` for any client-affecting change
* You can update client code & hot.js on-the-fly with `node server/kill.js`

Dependencies:



+ 1
- 16
builder.js View File

@@ -21,23 +21,9 @@ var reload_state = _.debounce(function () {
server.kill('SIGHUP');
}, 500);

var build_client = _.debounce(function () {
var makeBin = config.GNU_MAKE || '/usr/bin/make';
var make = child_process.execFile(makeBin, ['-s', '-q', 'client']);
make.once('exit', function (code) {
if (!code)
return;
console.log('make client');
var make = child_process.execFile(makeBin, ['-s', 'client']);
make.stdout.pipe(process.stdout);
make.stderr.pipe(process.stderr);
});
}, 500);

deps.SERVER_DEPS.forEach(monitor.bind(null, start_server));
deps.SERVER_STATE.forEach(monitor.bind(null, reload_state));
deps.CLIENT_DEPS.forEach(monitor.bind(null, build_client));
monitor(build_client, 'config.js');
deps.CLIENT_DEPS.forEach(monitor.bind(null, reload_state));

function monitor(func, dep) {
var mtime = new Date;
@@ -50,4 +36,3 @@ function monitor(func, dep) {
}

start_server();
build_client();

+ 6
- 2
deps.js View File

@@ -2,12 +2,15 @@ var config = require('./config');

var minJs = config.DEBUG ? '.js' : '.min.js';

exports.CLIENT_DEPS = [
exports.VENDOR_DEPS = [
'lib/yepnope' + minJs,
'lib/underscore' + minJs,
'lib/backbone' + minJs,
'lib/oninput' + minJs,
'lib/jquery.cookie' + minJs,
];

exports.CLIENT_DEPS = [
'common.js',
'client/init.js',
'client/models.js',
@@ -37,8 +40,9 @@ exports.SERVER_DEPS = [
'db.js',
'deps.js',
'etc.js',
'get.js',
'hooks.js',
'make_client.js',
'pipeline.js',
'tail.js',
'curfew/server.js',
'lib/underscore.js',


+ 11
- 0
etc.js View File

@@ -34,6 +34,17 @@ Muggle.prototype.deepest_reason = function () {
return this;
};

exports.move = function (src, dest, callback) {
child_process.execFile('/bin/mv', ['--', src, dest],
function (err, stdout, stderr) {
if (err)
callback(Muggle("Couldn't move file into place.",
stderr || err));
else
callback(null);
});
};

exports.movex = function (src, dest, callback) {
child_process.execFile('/bin/mv', ['-n', '--', src, dest],
function (err, stdout, stderr) {


+ 0
- 36
get.js View File

@@ -1,36 +0,0 @@
function get_version(deps, callback) {
require('child_process').exec('git log -1 --format=%h '+deps.join(' '),
function (err, stdout, stderr) {
if (err)
callback(err);
else
callback(null, stdout.trim());
});
}
exports.get_version = get_version;

if (process.argv[1] == __filename) {
if (process.argv.length != 3) {
console.error("Specify a config key or --client-version.");
process.exit(-1);
}
var arg = process.argv[2];
var deps = require('./deps');
if (arg == '--client-version') {
get_version(deps.CLIENT_DEPS, function (err, version) {
if (err)
throw err;
else
console.log(version);
});
}
else if (arg in deps) {
console.log(deps[arg].join(' '));
}
else {
var config = require('./config');
if (!(arg in config))
throw "No such config value " + arg;
console.log(config[arg]);
}
}

+ 13
- 18
make_client.js View File

@@ -23,16 +23,8 @@ function lookup_config(key) {
var config_re = /\b(\w+onfig)\.(\w+)\b/;

function convert(file, cb) {
if (/^lib\//.test(file)) {
fs.readFile(file, 'UTF-8', function (err, lib) {
if (err)
return cb(err);
out.write(lib);
out.write('\n');
cb(null);
});
return;
}
if (/^lib\//.test(file))
return cb("lib/* should be in VENDOR_DEPS");
if (/^config\.js/.test(file))
return cb("config.js shouldn't be in client");

@@ -138,6 +130,15 @@ function make_minified(files, out, cb) {

exports.make_minified = make_minified;

function make_maybe_minified(files, out, cb) {
if (config.DEBUG)
make_client(files, out, cb);
else
make_minified(files, out, cb);
}

exports.make_maybe_minified = make_maybe_minified;

if (require.main === module) {
var files = [];
for (var i = 2; i < process.argv.length; i++) {
@@ -152,13 +153,7 @@ if (require.main === module) {
}
}

var out;
if (config.DEBUG)
make_client(files, process.stdout, handler);
else
make_minified(files, process.stdout, handler);

function handler(err) {
make_maybe_minified(files, process.stdout, function (err) {
if (err) throw err;
}
});
}

+ 163
- 0
pipeline.js View File

@@ -0,0 +1,163 @@
var async = require('async'),
config = require('./config'),
crypto = require('crypto'),
etc = require('./etc'),
fs = require('fs'),
make_client = require('./make_client').make_maybe_minified,
pathJoin = require('path').join,
stream = require('stream'),
tmp_file = require('tmp').file,
util = require('util');

const PUBLIC_JS = pathJoin('www', 'js');

function HashingStream(out) {
stream.Writable.call(this);

this._hash = crypto.createHash('MD5');
this._outStream = out;
}
util.inherits(HashingStream, stream.Writable);

HashingStream.prototype._write = function (chunk, encoding, cb) {
this._hash.update(chunk);
this._outStream.write(chunk, encoding, cb);
};

HashingStream.prototype.end = function (cb) {
if (arguments.length > 1)
throw new Error("TODO multi-arg HashingStream.end");
var self = this;
stream.Writable.prototype.end.call(this, function () {
self._outStream.end(function () {
if (cb)
cb();
});
});
};

function end_and_move_js(stream, dir, prefix, cb) {
stream.end(function () {
var fnm;
if (config.DEBUG) {
fnm = prefix + '-debug.js';
}
else {
var hash = stream._hash.digest('hex').slice(0, 10);
fnm = prefix + '-' + hash + '.min.js';
}
var tmp = stream._tmpFilename;
etc.move(tmp, pathJoin(dir, fnm), function (err) {
if (err)
return cb(err);
cb(null, fnm);
});
});
};


function make_hashing_stream(cb) {
// ideally the stream would be returned immediately and handle
// this step internally...
tmp_file({dir: '.build', postfix: '.gen.js'}, function (err, tmp, fd) {
if (err)
return cb(err);
var out = fs.createWriteStream(null, {fd: fd});
out.once('error', cb);

if (config.DEBUG) {
out._tmpFilename = tmp;
cb(null, out);
}
else {
var stream = new HashingStream(out);
stream._tmpFilename = tmp;
cb(null, stream);
}
});
}

function build_vendor_js(cb) {
var deps = require('./deps');
make_hashing_stream(function (err, stream) {
if (err)
return cb(err);
async.eachSeries(deps.VENDOR_DEPS, function (file, cb) {
fs.readFile(file, function (err, buf) {
if (err)
return cb(err);
stream.write(buf, cb);
});
}, function (err) {
if (err)
return cb(err);
end_and_move_js(stream, PUBLIC_JS, 'vendor', cb);
});
});
}

function build_client_js(cb) {
var deps = require('./deps');
make_hashing_stream(function (err, stream) {
if (err)
return cb(err);
make_client(deps.CLIENT_DEPS, stream, function (err) {
if (err)
return cb(err);
end_and_move_js(stream, PUBLIC_JS, 'client', cb);
});
});
}

function build_mod_client_js(cb) {
var deps = require('./deps');
make_hashing_stream(function (err, stream) {
if (err)
return cb(err);
make_client(deps.MOD_CLIENT_DEPS, stream, function (err) {
if (err)
return cb(err);
end_and_move_js(stream, 'state', 'mod', cb);
});
});
}

function commit_assets(metadata, cb) {
tmp_file({dir: '.build', postfix: '.json'}, function (err, tmp, fd) {
if (err)
return cb(err);
var stream = fs.createWriteStream(null, {fd: fd});
stream.once('error', cb);
stream.end(JSON.stringify(metadata) + '\n', function () {
etc.move(tmp, pathJoin('state', 'scripts.json'), cb);
});
});
}

function rebuild(cb) {
etc.checked_mkdir('state', function (err) {
etc.checked_mkdir('.build', function (err) {
if (err) return cb(err);
async.parallel({
vendor: build_vendor_js,
client: build_client_js,
mod: build_mod_client_js,
}, function (err, hashes) {
if (err)
return cb(err);
commit_assets(hashes, cb);
});
});
});
}
exports.rebuild = rebuild;

exports.refresh_deps = function () {
delete require.cache[pathJoin(__dirname, 'deps.js')];
};

if (require.main === module) {
rebuild(function (err) {
if (err) throw err;
});
}

+ 9
- 3
server/server.js View File

@@ -7,7 +7,6 @@ var _ = require('../lib/underscore'),
config = require('../config'),
db = require('../db'),
fs = require('fs'),
get_version = require('../get').get_version,
hooks = require('../hooks'),
imager = require('../imager'),
Muggle = require('../etc').Muggle,
@@ -252,8 +251,15 @@ web.resource(/^\/(login|logout)\/$/, function (req, params, cb) {
});

function write_mod_js(resp, ident) {
resp.writeHead(200, {
'Content-Type': 'text/javascript; charset=UTF-8'});
if (!RES.modJs) {
resp.writeHead(500);
resp.end('Mod js not built?!');
return;
}

var noCacheJs = _.clone(web.noCacheHeaders);
noCacheJs['Content-Type'] = 'text/javascript; charset=UTF-8';
resp.writeHead(200, noCacheJs);
resp.write('(function (IDENT) {');
resp.write(RES.modJs);
resp.end('})(' + JSON.stringify(ident) + ');');


+ 40
- 23
server/state.js View File

@@ -1,12 +1,11 @@
var _ = require('../lib/underscore'),
async = require('async'),
child_process = require('child_process'),
config = require('../config'),
crypto = require('crypto'),
fs = require('fs'),
get_version = require('../get').get_version,
hooks = require('../hooks'),
path = require('path'),
pipeline = require('../pipeline'),
vm = require('vm');

_.templateSettings = {
@@ -57,8 +56,38 @@ function reload_hot_config(cb) {
});
}

function reload_scripts(cb) {
var json = path.join('state', 'scripts.json');
fs.readFile(json, 'UTF-8', function (err, json) {
if (err)
cb(err);
var js;
try {
js = JSON.parse(json);
}
catch (e) {
return cb(e);
}
if (!js || !js.vendor || !js.client)
return cb('Bad state/scripts.json.');

HOT.VENDOR_JS = js.vendor;
HOT.CLIENT_JS = js.client;

var modJs = path.join('state', js.mod);
fs.readFile(modJs, 'UTF-8', function (err, modSrc) {
if (err)
return cb(err);
RES.modJs = modSrc;
cb(null);
});
});
}

function reload_resources(cb) {

var deps = require('../deps');

function read(dir, file) {
return fs.readFile.bind(fs, path.join(dir, file), 'UTF-8');
}
@@ -71,7 +100,6 @@ function reload_resources(cb) {
src: expanded};
}
async.parallel({
version: get_version.bind(null, deps.CLIENT_DEPS),
index: read('tmpl', 'index.html'),
filter: read('tmpl', 'filter.html'),
login: read('tmpl', 'login.html'),
@@ -80,14 +108,9 @@ function reload_resources(cb) {
aLookup: read('tmpl', 'alookup.html'),
notFound: read('www', '404.html'),
serverError: read('www', '50x.html'),
modJs: make_mod_js,
}, function (err, res) {
if (err)
return cb(err);
if (config.DEBUG)
config.CLIENT_JS = 'client.debug.js';
else
config.CLIENT_JS = 'client-' + res.version + '.js';

var index = tmpl(res.index);
RES.indexTmpl = index.tmpl;
@@ -102,13 +125,20 @@ function reload_resources(cb) {
RES.aLookupHtml = res.aLookup;
RES.notFoundHtml = res.notFound;
RES.serverErrorHtml = res.serverError;
RES.modJs = res.modJs;
hooks.trigger('reloadResources', RES, cb);
});
}

exports.reload_hot_resources = function (cb) {
async.series([reload_hot_config, reload_resources], cb);
pipeline.refresh_deps();

async.series([
reload_hot_config,
pipeline.rebuild,
reload_scripts,
reload_resources,
], cb);
}

function make_navigation_html() {
@@ -126,19 +156,6 @@ function make_navigation_html() {
return bits.join('');
}

function make_mod_js(cb) {
var makeBin = config.GNU_MAKE || '/usr/bin/make';
var cmd = makeBin + ' -s modjs';
child_process.exec(cmd, function (err, stdout, stderr) {
if (err)
cb(err);
else if (stderr && stderr.trim())
cb(stderr.trim());
else
cb(null, stdout);
});
}

function read_exits(file, cb) {
fs.readFile(file, 'UTF-8', function (err, lines) {
if (err)


+ 1
- 0
tmpl/index.html View File

@@ -20,4 +20,5 @@ $THREADS
$NAV
<script src="{{MEDIA_URL}}js/jquery-1.10.2.min.js"></script>
<script src="{{MEDIA_URL}}js/sockjs-0.3.4.min.js"></script>
<script src="{{MEDIA_URL}}js/{{VENDOR_JS}}"></script>
<script src="{{MEDIA_URL}}js/{{CLIENT_JS}}"></script>

Loading…
Cancel
Save