diff --git a/.gitignore b/.gitignore index 3c3629e..8c8d22b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ node_modules +.tm_properties diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0bdf590 --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +test: + @mocha + +spec: + @mocha -R spec diff --git a/Readme.md b/Readme.md index 401fc59..849d626 100644 --- a/Readme.md +++ b/Readme.md @@ -1,6 +1,6 @@ # kwik-e-mon -Monitor one-off tasks on your servers. +Monitor one-off tasks on your servers using Redis. ## Installation @@ -10,13 +10,111 @@ npm install -g kwikemon ## Usage -I can't think of a concise example use of the command line tool. Here's how it works: + $ kwikemond & + $ curl -s localhost/nginx_status | grep Active | kwikemon nginx-connections + $ curl localhost:1111/nignx-connections + Active connections: 316 + $ kwikemon foo bar + $ curl localhost:1111/ + foo: bar + nginx-connections: Active connections: 316 -- you continuously pipe data to `kwikemon ` on stdin +Here's how it works: + +- call `kwikemon thing status` to set the text for the monitor named "thing" +- fire up the server, `kwikemond`, that serves up these monitors in a big list or individually + +Alternatively: + +- continuously pipe data to `kwikemon ` on stdin - every time a full line of text is received on stdin it becomes the new status for -- there's a simple web server, `kwikemond`, that serves up these monitors in a big list or individually -This is very much a work in progress and as the functionality is fleshed out this readme will improve as well. +To see everything `kwikemon` can do run it without arguments. + + # or with -h or --help + $ kwikemon + +This is very much a work in progress. + + +## API + +You can use kwikemon as a library. + + var kwikemon = require('kiwkemon'); + + kwikemon.set('foo', 'bar', function(err) { + kwikemon.fetch('foo', function(err, text) { + console.log('foo = ' + text); + }); + }); + + kwikemon.fetchAll(function(err, monitors) { + Object.keys(monitors).forEach(function (name) { + console.log(name + ' = ' + monitors[name]); + }); + }); + +Monitors expire 1 day after the last time they were set by default. You can pass in any `ttl` you +want though. + + // never expire + kwikemon.set('foo', 'bar', { ttl: 0 }); + + +## Protocol + +All kwikemon does is shove things into Redis in a standard way and read them out later. + +A monitor named `nginx` stores its data in the hash `kwikemon:monitor:nginx`. Hash fields +are: + + - text + - created + - modified + - updates + +The list of all monitors is a set stored at `kwikemon:monitors`. + +#### List + +This is when you should clean out expired entries. + + names = redis.smembers("kwikemon:monitors") + monitors = {} + for name in names: + if redis.exists("kwikemon:monitor:$name"): + monitors[name] = redis.hget("kwikemon:monitor:$name", "text") + return monitors + +#### Read + + redis.hgetall("kwikemon:monitor:nginx") + +#### Update + + exists = redis.exists("kwikemon:monitor:nginx") + if exists: + redis.hmset("kwikemon:monitor:nginx", { + text: "Active connections: 583" + modified: 1370668341943 + }) + redis.hincrby("kwikemon:monitor:nginx", "updates", 1) + else: + redis.hmset("kwikemon:monitor:nginx", { + text: "Active connections: 316" + updates: 1 + created: 1370668301943 + modified: 1370668301943 + }) + redis.sadd("kwikemon:monitors", "nginx") + # optional + redis.expire("kwikemon:monitor:nginx", ) + +#### Delete + + redis.del("kwikemon:monitor:nginx") + redis.srem("kwikemon:monitors", "nginx") ## License diff --git a/bin/kwikemon b/bin/kwikemon index 8c41e85..162c0a3 100755 --- a/bin/kwikemon +++ b/bin/kwikemon @@ -1,10 +1,121 @@ #!/usr/bin/env node -var kwikemon = require('../kwikemon.js') - , name = process.argv[2] - ; +var kwikemon = require('../kwikemon.js'); -process.stdin.pipe(kwikemon.createWriter(name)); -process.stdin.on('end', function() { - process.exit(0); -}); +function usage() { + console.log("usage: kwikemon [options]"); + console.log(" kwikemon [text]"); + console.log(); + console.log("options:"); + console.log(" -c, --clear remove all monitors"); + console.log(" -f, --fetch show the text of the named monitor"); + console.log(" -n, --count count monitors"); + console.log(" -h, --help show what you're reading now"); + console.log(" -l, --list show all monitors"); + console.log(" -r, --remove remove the named monitor"); + console.log(" -s, --sweep clean up expired & deleted monitors"); + console.log(" -t, --ttl show the TTL of the named monitor"); +} + +var opt = process.argv[2] + , name = process.argv[2] + , text = process.argv[3] + ; +if (opt && opt[0] == '-') { + switch (opt) { + case '-c': + case '--clear': + kwikemon.removeAll(function(err) { + process.exit(0); + }); + break; + + case '-f': + case '--fetch': + var name = process.argv[3]; + kwikemon.fetch(name, function(err, mon) { + if (mon) { + console.log(mon.text); + process.exit(0); + } + else { + console.log('error: no monitor named', name); + process.exit(1); + } + }); + break; + + case '-l': + case '--list': + kwikemon.fetchAll(function(err, monitors) { + Object.keys(monitors).forEach(function(name) { + console.log(name + ':', monitors[name].text); + }); + process.exit(0); + }); + break; + + case '-n': + case '--count': + kwikemon.count(function(err, n) { + console.log(n); + process.exit(0); + }); + break; + + case '-r': + case '--remove': + name = process.argv[3]; + if (name) { + kwikemon.remove(name, function(err) { + process.exit(0); + }) + } + else { + console.log("error: --remove requires a name"); + process.exit(1); + } + break; + + case '-s': + case '--sweep': + kwikemon.sweep(function(err) { + process.exit(0); + }); + break; + + case '-t': + case '--ttl': + var name = process.argv[3]; + kwikemon.fetchTTL(name, function(err, ttl) { + if (typeof ttl == 'number') { + console.log(ttl); + process.exit(0); + } + else { + console.log('error: no monitor named', name); + process.exit(1); + } + }); + break; + + default: + usage(); + process.exit(name == '-h' || name == '--help' ? 0 : 1); + } +} +else if (name && text) { + kwikemon.set(name, text, function() { + process.exit(0); + }); +} +else if (name) { + process.stdin.pipe(kwikemon.createWriter(name)); + process.stdin.on('end', function() { + process.exit(0); + }); +} +else { + usage(); + process.exit(1); +} diff --git a/kwikemon.js b/kwikemon.js index 8c75ec4..8488fc5 100644 --- a/kwikemon.js +++ b/kwikemon.js @@ -1,72 +1,155 @@ // Copyright 2013 Sami Samhuri module.exports = { - // read - fetchMonitor: fetchMonitor -, fetchMonitors: fetchMonitors // write -, monitor: monitor -, createWriter: createWriter + set: callbackOptional(set) +, writer: writer + + // read +, exists: callbackOptional(exists) +, fetch: callbackOptional(fetch) +, fetchTTL: callbackOptional(fetchTTL) +, list: list +, fetchAll: fetchAll +, count: count + + // remove +, remove: callbackOptional(remove) +, removeAll: removeAll +, sweep: sweep + + // change redis client +, redis: setRedis }; -var redis = require('redis').createClient() +var async = require('async') + , redis = require('redis').createClient() , LineEmitter = require('./line_emitter.js') ; -function monitor(name, text, options) { - console.log(name,'=',text) - options = options || {}; - if (typeof options == 'function') { - options = { cb: options }; - } - var key = 'kwikemon:monitor:' + name - , timeout = options.timeout || 86400 - ; - console.log('set',key,text) - redis.set(key, text, function(err, status) { - console.log('set',key,text) - if (err) throw err; - if (timeout >= 0) { - redis.expire(key, timeout); +function setRedis(newRedis) { + if (redis) redis.end(); + redis = newRedis; +} + +// Make the callback argument of a function optional. +// If the callback is passed it will call the function +// normally. If the callback isn't given a function +// that accepts the callback is returned, with the +// rest of the arguments fixed (like bind). +// +// function fetch(id, cb) { db.fetch(id, cb); } +// fetch = callbackOptional(fetch); +// +// function print(err, x) { if (err) throw err; console.log(x); } +// +// fetch(1, print); +// +// var fetch1 = fetch(1); +// fetch1(print); +function callbackOptional(fn, ctx) { + return function() { + var args = Array.prototype.slice.call(arguments); + var cb = args[args.length - 1]; + if (typeof cb == 'function') { + fn.apply(ctx, arguments); } - redis.sadd('kwikemon:monitors', name, function(err, status) { - if (options.cb) options.cb(); - }); + else { + return function(cb) { + args.push(cb); + fn.apply(ctx, args); + }; + } + }; +} + +function k(name) { + return 'kwikemon:monitor:' + name; +} + +function exists(name, cb) { + redis.exists(k(name), function(err, exists) { + if (err) return cb(err); + cb(null, exists == 1); }); } -function createWriter(name) { +// options: +// - ttl: time to live in seconds, <= 0 to never expire +function set(name, text, options, cb) { + if (typeof options == 'function') { + cb = options; + options = null; + } + options = options || {}; + var key = k(name) + , ttl = ('ttl' in options) ? options.ttl : 86400 + ; + exists(name, function(err, exists) { + var fields = { + text: text + , modified: Date.now() + } + , multi = redis.multi() + ; + if (!exists) { + fields.created = Date.now(); + } + multi + .hmset(key, fields) + .hincrby(key, 'updates', 1); + if (ttl != null) { + multi.expire(key, ttl); + } + multi.sadd('kwikemon:monitors', name); + multi.exec(cb); + }); +} + +function writer(name) { var le = new LineEmitter(); le.on('line', function(line) { - monitor(name, line); + set(name, line, function(err) { + if (err) throw err; + le.emit('monitor', name, line); + }); }); return le; } -function fetchMonitor(name, cb) { - redis.get('kwikemon:monitor:' + name, cb); +function fetch(name, cb) { + redis.hgetall(k(name), cb); } -function fetchMonitors(cb) { - var monitors = {} - , i = 0 +function fetchTTL(name, cb) { + redis.ttl(k(name), cb); +} + +function count(cb) { + redis.scard('kwikemon:monitors', cb); +} + +function sweep(cb) { + var i = 0 , n , checkIfDone = function() { i += 1; - if (i == n) cb(null, monitors); + if (i == n) cb(); } ; redis.smembers('kwikemon:monitors', function(err, names) { if (err) return cb(err); n = names.length; + if (n == 0) return cb(); names.forEach(function(name) { - fetchMonitor(name, function(err, text) { + exists(name, function(err, exists) { if (err) { - // missing? probably don't care + // meh, ignore it } - else { - monitors[name] = text; + // remove expired monitors + else if (!exists) { + remove(name); } checkIfDone(); }); @@ -74,3 +157,48 @@ function fetchMonitors(cb) { }); } +function list(cb) { + sweep(function(err) { + if (err) return cb(err); + redis.smembers('kwikemon:monitors', cb); + }); +} + +function fetchAll(cb) { + var monitors = {}; + list(function(err, names) { + if (err) return cb(err); + var fetchers = names.sort().map(function(name) { + return function(done) { + fetch(name, function(err, text) { + if (err) return done(err); + monitors[name] = text; + done(); + }); + }; + }); + async.parallel(fetchers, function(err, _) { + if (err) return cb(err); + cb(null, monitors) + }); + }); +} + +function remove(name, cb) { + redis.multi() + .del(k(name)) + .srem('kwikemon:monitors', name) + .exec(cb); +} + +function removeAll(cb) { + redis.smembers('kwikemon:monitors', function(err, names) { + if (err) return cb(err); + var multi = redis.multi(); + names.forEach(function(name) { + multi.del(k(name)); + multi.srem('kwikemon:monitors', name); + }); + multi.exec(cb); + }); +} diff --git a/package.json b/package.json index 6ea2b64..8c8fdb1 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,11 @@ , "dependencies": { "paramify": "0.0.x" , "redis": "0.8.x" + , "async": "0.2.x" + } +, "devDependencies": { + "mocha": "1.10.x" + , "better-assert": "1.0.x" } , "engines": { "node": ">=0.10" diff --git a/server.js b/server.js index 17f3fcf..27218dc 100644 --- a/server.js +++ b/server.js @@ -31,9 +31,17 @@ function stop() { } function handleRequest(req, res) { - var name = req.url.replace(/^\//, ''); + var name = req.url.replace(/^\//, '') + , type = 'html' + , m + ; + if (name == 'favicon.ico') return res.end(); + if (m = name.match(/\.(json|txt)$/)) { + type = m[1]; + name = name.replace(RegExp('\.' + type + '$'), ''); + } if (name) { - kwikemon.fetchMonitor(name, function(err, text) { + kwikemon.fetch(name, function(err, text) { if (err) { res.end('error: ' + (err.message || 'unknown')); return; @@ -43,7 +51,7 @@ function handleRequest(req, res) { } // all else { - kwikemon.fetchMonitors(function(err, monitors) { + kwikemon.fetchAll(function(err, monitors) { if (err) { res.end('error: ' + (err.message || 'unknown')); return; diff --git a/test.js b/test.js new file mode 100755 index 0000000..fd502af --- /dev/null +++ b/test.js @@ -0,0 +1,213 @@ +// Copyright 2013 Sami Samhuri + +var assert = require('better-assert') + , async = require('async') + , kwikemon = require('./kwikemon') + , redis = require('redis').createClient() + ; + +before(function(done) { + redis.select(1, function() { + kwikemon.redis(redis); + done(); + }); +}); +beforeEach(kwikemon.removeAll); +after(kwikemon.removeAll); + +describe("kwikemon", function() { + describe("#set", function() { + it("should set text", function(done) { + kwikemon.set('foo', 'bar', function(err) { + kwikemon.fetch('foo', function(err, mon) { + assert(mon.text == 'bar'); + done(); + }); + }); + }); + + it("should overwrite text", function(done) { + kwikemon.set('foo', 'baz', function(err) { + kwikemon.fetch('foo', function(err, mon) { + assert(mon.text == 'baz'); + done(); + }); + }); + }); + + it("should set custom ttls", function(done) { + kwikemon.set('foo', 'bar', { ttl: 1 }, function(err) { + kwikemon.fetchTTL('foo', function(err, ttl) { + assert(ttl <= 1); + done(); + }); + }) + }); + + it("should not expire with a ttl of zero", function(done) { + kwikemon.set('foo', 'bar', { ttl: 0 }, function(err) { + kwikemon.fetchTTL('foo', function(err, ttl) { + assert(ttl == -1); + done(); + }); + }); + }); + + it("should not expire when ttl is < 0", function(done) { + kwikemon.set('foo', 'bar', { ttl: -1 }, function(err) { + kwikemon.fetchTTL('foo', function(err, ttl) { + assert(ttl == -1); + done(); + }); + }) + }); + }); + + describe("#writer", function() { + it("should monitor each line of text written", function(done) { + var writer = kwikemon.writer('foo'); + writer.once('monitor', function(name, text) { + assert(text == 'a'); + writer.once('monitor', function(name, text) { + assert(text == 'b'); + done(); + }); + writer.write("b\n"); + }); + writer.write("a\n"); + }); + + it("should only monitor complete lines of text", function(done) { + var writer = kwikemon.writer('foo'); + writer.once('monitor', function(name, text) { + assert(text == 'complete'); + writer.once('monitor', function(name, text) { + assert(text == 'incomplete'); + done(); + }); + writer.write("plete\n"); + }); + writer.write("complete\n"); + writer.write("incom"); + }); + }); + + describe("#fetch", function() { + it("should fetch the last text monitored", function(done) { + async.series([ + kwikemon.set('foo', 'bar') + , kwikemon.set('foo', 'marcellus') + , kwikemon.fetch('foo') + ], + function(err, results) { + var mon = results[2]; + assert(mon.text == 'marcellus'); + done(); + } + ); + }); + + it("should fetch null for non-existent monitors", function(done) { + kwikemon.fetch('non-existent', function(err, mon) { + assert(mon == null); + done(); + }); + }); + }); + + describe("#fetchTTL", function() { + it("should fetch the last TTL set", function(done) { + kwikemon.set('foo', 'bar', { ttl: 300 }, function(err) { + kwikemon.fetchTTL('foo', function(err, ttl) { + assert(ttl <= 300); + done(); + }); + }); + }); + + it("should return -1 for non-existent monitors", function(done) { + kwikemon.fetchTTL('non-existent', function(err, ttl) { + assert(ttl == -1); + done(); + }); + }); + }); + + describe("#fetchAll", function() { + it("should fetch all monitors", function(done) { + async.series([ + kwikemon.set('a', '1') + , kwikemon.set('b', '2') + , kwikemon.set('c', '3') + , kwikemon.fetchAll + ], + function(err, results) { + var monitors = results.pop() + , names = Object.keys(monitors).sort(); + assert(names.length == 3); + assert(names[0] == 'a' && monitors.a.text == '1'); + assert(names[1] == 'b' && monitors.b.text == '2'); + assert(names[2] == 'c' && monitors.c.text == '3'); + done(); + } + ); + }); + }); + + describe("#count", function() { + it("should count all monitors", function(done) { + async.series([ + kwikemon.set('a', '1') + , kwikemon.set('b', '2') + , kwikemon.set('c', '3') + , kwikemon.count + ], + function(err, results) { + var n = results.pop(); + assert(n == 3); + done(); + } + ); + }); + }); + + describe("#remove", function() { + it("should remove the named monitor", function(done) { + async.series([ + kwikemon.set('foo', 'bar') + , kwikemon.remove('foo') + , kwikemon.exists('foo') + ], + function(err, results) { + var exists = results.pop(); + assert(!exists); + done(); + } + ); + }); + }); + + describe("#removeAll", function() { + it("should remove the named monitor", function(done) { + async.series([ + kwikemon.set('foo', 'bar') + , kwikemon.set('baz', 'quux') + , kwikemon.removeAll + , kwikemon.exists('foo') + , kwikemon.exists('baz') + , kwikemon.count + ], + function(err, results) { + var n = results.pop() + , bazExists = results.pop() + , fooExists = results.pop() + ; + assert(!fooExists); + assert(!bazExists); + assert(n == 0); + done(); + } + ); + }); + }); +});