Browse Source

try asyncing file read, and use readableStream so we can pipe the output as soon as possible

create package.json

debug

we need to pass end: false to response while we're combining streams, otherwise it'll end when the first close event moves through. also, create a seperate decryptor each time, although maybe it'll work if we pass end: false to it as well.

streamline operations a bit

trying to address potential memory leak

some cleanup

Update package.json

Use a more appropriate version number based on statements in docs and elsewhere.

async writes

don't trigger response.end before everything may be done. delegate to the pipe and iteration

update compress_blocks so it's all async

load fs when log is initially loaded, not on every invocation

only use async methods so we can constuct a common procedure for local or remote file systems, and modularize code for testing and sanity

start building out tests

replace test file with truncated file

truncate test mp3

update interface for file type analysis

create common hash method and move wave offset method into module for testing

switch to library hashing method and chai expectations

test for wav offsetting

switch to utility method for wav offset

include chai in dev dependencies

cleanup

add tests around loading and saving inodes

comment

tweaks

create lib methods for handling params and target urls, for test coverage and standardization

build out gcs operations

guard around undefined getMaxListeners
main
Marc Brakken 5 years ago
parent
commit
ddf945143d
  1. 1
      .gitignore
  2. 3
      jlog.js
  3. 88
      lib/file-types.js
  4. 10
      lib/fs/disk-operations.js
  5. 35
      lib/google-cloud-storage/disk-operations.js
  6. 153
      lib/inode.js
  7. 41
      lib/static.js
  8. 223
      lib/utils.js
  9. 39
      lib/validate.js
  10. 47
      package.json
  11. 940
      server.js
  12. 65
      test/file-types.js
  13. BIN
      test/fixtures/test.mp3
  14. BIN
      test/fixtures/test.wav
  15. 237
      test/utils.js
  16. 121
      test/validate.js
  17. 64
      tools/compress_blocks.js

1
.gitignore

@ -3,3 +3,4 @@ config.js
blocks
*swp
.DS_Store
*.sublime-*

3
jlog.js

@ -1,3 +1,5 @@
var fs = require("fs");
module.exports = {
DEBUG: 0,
INFO: 1,
@ -6,7 +8,6 @@ module.exports = {
level: 0, // default log level
path: null, // default is, don't log to a file
message: function(severity, log_message){
var fs = require("fs");
if(severity >= this.level){
console.log(Date() + "\t" + Math.round((process.memoryUsage().rss/1024)/1024) + "MB\t" + severity + "\t" + log_message);
if(this.path){

88
lib/file-types.js

@ -0,0 +1,88 @@
"use strict";
/* globals module */
/**
Lookup WAVE format by chunk size.
Chunk size of 16 === PCM (1)
Chunk size of 40 === WAVE_FORMAT_EXTENSIBLE (65534)
The WAVE_FORMAT_EXTENSIBLE format should be used whenever:
PCM data has more than 16 bits/sample.
The number of channels is more than 2.
The actual number of bits/sample is not equal to the container size.
The mapping from channels to speakers needs to be specified.
We should probably do more finer-grained analysis of this format for determining duration,
by examining any fact chunk between the fmt chunk and the data,but this should be enough for
current use cases.
Chunk size of 18 === non-PCM (3, 6, or 7)
This could be IEEE float (3), 8-bit ITU-T G.711 A-law (6), 8-bit ITU-T G.711 µ-law (7),
all of which probably require different calculations for duration and are not implemented
Further reading: http://www-mmsp.ece.mcgill.ca/Documents/AudioFormats/WAVE/WAVE.html
*/
var WAVE_FMTS = {
16 : 1,
40 : 65534
};
function format_from_block(block){
if (is_wave(block)) {
return "wave";
} else {
return "unknown";
}
}
function is_wave(block){
return block.toString("utf8", 0, 4) === "RIFF" &&
block.toString("utf8", 8, 12) === "WAVE" &&
WAVE_FMTS[block.readUInt32LE(16)] == block.readUInt16LE(20);
}
var _analyze_type = {
wave: function(block, result){
var subchunk_byte = 36;
var subchunk_id = block.toString("utf8", subchunk_byte, subchunk_byte+4);
var block_length = block.length;
while (subchunk_id !== 'data' && subchunk_byte < block_length) {
// update start byte for subchunk by adding
// the size of the subchunk + 8 for the id and size bytes (4 each)
subchunk_byte = subchunk_byte + block.readUInt32LE(subchunk_byte+4) + 8;
subchunk_id = block.toString("utf8", subchunk_byte, subchunk_byte+4);
}
var subchunk_size = block.readUInt32LE(subchunk_byte+4);
var audio_data_size = subchunk_id === 'data' ? subchunk_size : result.size;
result.type = "wave";
result.size = block.readUInt32LE(4);
result.channels = block.readUInt16LE(22);
result.bitrate = block.readUInt32LE(24);
result.resolution = block.readUInt16LE(34);
result.subchunk_id = subchunk_id;
result.subchunk_byte = subchunk_byte;
result.data_block_size = block.readUInt16LE(32);
result.duration = (audio_data_size * 8) / (result.channels * result.resolution * result.bitrate);
return result;
},
unknown: function(block, result){
return result;
}
};
// examine the contents of a block to generate metadata
module.exports.analyze = function(block){
var result = { type: format_from_block(block) };
return _analyze_type[result.type](block, result);
};

10
lib/fs/disk-operations.js

@ -0,0 +1,10 @@
"use strict";
/* globals require, module */
var fs = require("fs");
module.exports.exists = fs.stat;
module.exports.read = fs.readFile;
module.exports.stream_read = fs.createReadStream;
module.exports.write = fs.writeFile;
module.exports.delete = fs.unlink;

35
lib/google-cloud-storage/disk-operations.js

@ -0,0 +1,35 @@
"use strict";
/* globals require, module */
var config = require("./config.js");
var gcs = require("@google-cloud/storage")(config.GOOGLE_CLOUD_STORAGE.AUTHENTICATION);
// {
// projectId: 'grape-spaceship-123',
// keyFilename: '/path/to/keyfile.json'
// }
var bucket = gcs.bucket(config.GOOGLE_CLOUD_STORAGE.BUCKET);
module.exports.read = function(file_path, callback){
return bucket.file(file_path).download(callback);
};
module.exports.exists = function(file_path, callback){
return bucket.file(file_path).getMetadata(callback);
};
module.exports.stream_read = function(file_path){
return bucket.file(file_path).createReadStream;
};
module.exports.write = function(file_path, contents /*[, options], cb */){
var args = Array.prototype.slice.call(arguments);
var callback = args.pop();
return bucket.file(file_path).save(contents, callback);
};
module.exports.delete = function(file_path, callback){
return bucket.file(file_path).delete(callback);
};

153
lib/inode.js

@ -0,0 +1,153 @@
"use strict";
/* globals require, module, Buffer */
var config = require("../config.js");
var log = require("../jlog.js");
var utils = require("./utils.js");
var file_types = require("./file-types.js");
// global to keep track of storage location rotation
var next_storage_location = 0;
var Inode = {
init: function(url){
this.input_buffer = new Buffer("");
this.block_size = config.BLOCK_SIZE;
this.file_metadata = {};
this.file_metadata.url = url;
this.file_metadata.created = (new Date()).getTime();
this.file_metadata.version = 0;
this.file_metadata.private = false;
this.file_metadata.encrypted = false;
this.file_metadata.fingerprint = null;
this.file_metadata.access_key = null;
this.file_metadata.content_type = "application/octet-stream";
this.file_metadata.file_size = 0;
this.file_metadata.block_size = this.block_size;
this.file_metadata.blocks_replicated = 0;
this.file_metadata.inode_replicated = 0;
this.file_metadata.blocks = [];
// create fingerprint to uniquely identify this file
this.file_metadata.fingerprint = utils.sha1_to_hex(this.file_metadata.url);
// use fingerprint as default key
this.file_metadata.access_key = this.file_metadata.fingerprint;
},
write: function(chunk, req, callback){
this.input_buffer = new Buffer.concat([this.input_buffer, chunk]);
if (this.input_buffer.length > this.block_size) {
req.pause();
this.process_buffer(false, function(result){
req.resume();
callback(result);
});
} else {
callback(true);
}
},
close: function(callback){
var self = this;
log.message(0, "flushing remaining buffer");
// update original file size
self.file_metadata.file_size = self.file_metadata.file_size + self.input_buffer.length;
self.process_buffer(true, function(result){
if(result){
// write inode to disk
utils.save_inode(self.file_metadata, callback);
}
});
},
process_buffer: function(flush, callback){
var self = this;
var total = flush ? 0 : self.block_size;
this.store_block(!flush, function(err/*, result*/){
if (err) {
log.message(log.DEBUG, "process_buffer result: " + err);
return callback(false);
}
if (self.input_buffer.length > total) {
self.process_buffer(flush, callback);
} else {
callback(true);
}
});
},
store_block: function(update_file_size, callback){
var self = this;
var chunk_size = this.block_size;
// grab the next block
var block = this.input_buffer.slice(0, chunk_size);
if(this.file_metadata.blocks.length === 0){
// grok known file types
var analysis_result = file_types.analyze(block);
log.message(log.INFO, "block analysis result: " + JSON.stringify(analysis_result));
// if we found out anything useful, annotate the object's metadata
this.file_metadata.media_type = analysis_result.type;
if(analysis_result.type != "unknown"){
this.file_metadata.media_size = analysis_result.size;
this.file_metadata.media_channels = analysis_result.channels;
this.file_metadata.media_bitrate = analysis_result.bitrate;
this.file_metadata.media_resolution = analysis_result.resolution;
this.file_metadata.media_duration = analysis_result.duration;
}
if (analysis_result.type === "wave") {
block = block.slice(0, utils.wave_audio_offset(block, analysis_result));
}
}
// if encryption is set, encrypt using the hash above
if(this.file_metadata.encrypted && this.file_metadata.access_key){
log.message(log.INFO, "encrypting block");
block = utils.encrypt(block, this.file_metadata.access_key);
} else {
// if even one block can't be encrypted, say so and stop trying
this.file_metadata.encrypted = false;
}
// store the block
var block_object = {};
// generate a hash of the block to use as a handle/filename
block_object.block_hash = utils.sha1_to_hex(block);
utils.commit_block_to_disk(block, block_object, next_storage_location, function(err, result){
if (err) {
return callback(err);
}
// increment (or reset) storage location (striping)
next_storage_location++;
if(next_storage_location === config.STORAGE_LOCATIONS.length){
next_storage_location = 0;
}
// update inode
self.file_metadata.blocks.push(result);
// update original file size
// we need to update filesize here due to truncation at the front,
// but need the check to avoid double setting during flush
// is there a better way?
if (update_file_size) {
self.file_metadata.file_size = self.file_metadata.file_size + chunk_size;
}
// advance buffer
self.input_buffer = self.input_buffer.slice(chunk_size);
return callback(null, result);
});
}
};
module.exports = Inode;

41
lib/static.js

@ -0,0 +1,41 @@
"use strict";
/* globals module */
module.exports.ALLOWED_METHODS = ["GET",
"POST",
"PUT",
"DELETE",
"OPTIONS"];
module.exports.ALLOWED_HEADERS = ["Accept",
"Accept-Version",
"Api-Version",
"Content-Type",
"Origin",
"Range",
"X_FILENAME",
"X-Access-Key",
"X-Access-Token",
"X-Append",
"X-Encrypted",
"X-Private",
"X-Replacement-Access-Key",
"X-Requested-With"];
module.exports.EXPOSED_HEADERS = ["X-Media-Bitrate",
"X-Media-Channels",
"X-Media-Duration",
"X-Media-Resolution",
"X-Media-Size",
"X-Media-Type"];
module.exports.ACCEPTED_PARAMS = ["access_key",
"access_token",
"block_only",
"content_type",
"encrypted",
"expires",
"inode_only",
"private",
"replacement_access_key",
"version"];

223
lib/utils.js

@ -0,0 +1,223 @@
"use strict";
/* globals require, module */
var crypto = require("crypto");
var path = require("path");
var url = require("url");
var log = require("../jlog.js");
var config = require("../config.js");
var operations = require("./" + (config.CONFIGURED_STORAGE || "fs") + "/disk-operations.js");
var TOTAL_LOCATIONS = config.STORAGE_LOCATIONS.length;
// simple encrypt-decrypt functions
module.exports.encrypt = function encrypt(data, key){
var cipher = crypto.createCipher("aes-256-cbc", key);
cipher.write(data);
cipher.end();
return cipher.read();
};
var sha1_to_hex = function sha1_to_hex(data){
var shasum = crypto.createHash("sha1");
shasum.update(data);
return shasum.digest("hex");
};
module.exports.sha1_to_hex = sha1_to_hex;
// save inode to disk
module.exports.save_inode = function save_inode(inode, callback){
var accessed_locations = 0;
var _cb = function _cb(error){
accessed_locations++;
if(error){
log.message(log.ERROR, "Error saving inode: " + error);
} else {
log.message(log.INFO, "Inode saved to disk");
}
if (accessed_locations === TOTAL_LOCATIONS) {
return callback(inode);
}
};
// store a copy of each inode in each storage location for redundancy
for(var storage_location in config.STORAGE_LOCATIONS){
var selected_location = config.STORAGE_LOCATIONS[storage_location];
operations.write(path.join(selected_location.path, inode.fingerprint + ".json"), JSON.stringify(inode), _cb);
}
};
// load inode from disk
module.exports.load_inode = function load_inode(uri, callback){
log.message(log.DEBUG, "uri: " + uri);
// calculate fingerprint
var inode_fingerprint = sha1_to_hex(uri);
var _load_inode = function _load_inode(idx){
var selected_path = config.STORAGE_LOCATIONS[idx].path;
log.message(log.DEBUG, "Loading inode from " + selected_path);
operations.read(path.join(selected_path, inode_fingerprint + ".json"), function(err, data){
idx++;
if (err) {
if (idx === TOTAL_LOCATIONS) {
log.message(log.WARN, "Unable to load inode for requested URL: " + uri);
return callback(err);
} else {
log.message(log.DEBUG, "Error loading inode from " + selected_path);
return _load_inode(idx);
}
}
try {
var inode = JSON.parse(data);
log.message(log.INFO, "Inode loaded from " + selected_path);
return callback(null, inode);
} catch(ex) {
if (idx === TOTAL_LOCATIONS) {
log.message(log.WARN, "Unable to parse inode for requested URL: " + uri);
return callback(ex);
} else {
log.message(log.DEBUG, "Error parsing inode from " + selected_path);
return _load_inode(idx);
}
}
});
};
_load_inode(0);
};
module.exports.commit_block_to_disk = function commit_block_to_disk(block, block_object, next_storage_location, callback) {
// if storage locations exist, save the block to disk
var total_locations = config.STORAGE_LOCATIONS.length;
if(total_locations > 0){
// check all storage locations to see if we already have this block
var on_complete = function on_complete(found_block){
// TODO: consider increasing found count to enable block redundancy
if(!found_block){
// write new block to next storage location
// TODO: consider implementing in-band compression here
var dir = config.STORAGE_LOCATIONS[next_storage_location].path;
operations.write(dir + block_object.block_hash, block, "binary", function(err){
if (err) {
return callback(err);
}
block_object.last_seen = dir;
log.message(log.INFO, "New block " + block_object.block_hash + " written to " + dir);
return callback(null, block_object);
});
} else {
log.message(log.INFO, "Duplicate block " + block_object.block_hash + " not written to disk");
return callback(null, block_object);
}
};
var locate_block = function locate_block(idx){
var location = config.STORAGE_LOCATIONS[idx];
var file = location.path + block_object.block_hash;
idx++;
operations.exists(file + ".gz", function(err, result){
if (result) {
log.message(log.INFO, "Duplicate compressed block " + block_object.block_hash + " found in " + location.path);
block_object.last_seen = location.path;
return on_complete(true);
} else {
operations.exists(file, function(err_2, result_2){
if (err_2) {
log.message(log.INFO, "Block " + block_object.block_hash + " not found in " + location.path);
}
if (result_2) {
log.message(log.INFO, "Duplicate block " + block_object.block_hash + " found in " + location.path);
block_object.last_seen = location.path;
return on_complete(true);
} else {
if (idx >= total_locations) {
return on_complete(false);
} else {
locate_block(idx);
}
}
});
}
});
};
locate_block(0);
} else {
log.message(log.WARN, "No storage locations configured, block not written to disk");
return callback(null, block_object);
}
};
// Use analyze data to identify offset until non-zero audio, grab just that portion to store.
// In analyze we identified the "data" starting byte and block_align ((Bit Size * Channels) / 8)
// We'll start the scan at block.readUInt32LE([data chunk offset] + 8) in order to find the
// start of non-zero audio data, and slice off everything before that point as a seperate block.
// That way we can deduplicate tracks with slightly different silent leads.
module.exports.wave_audio_offset = function wave_audio_offset(block, data, default_size){
// block_align most likely to be 4, but it'd be nice to handle alternate cases.
// Essentially, we should use block["readUInt" + (block_align * 8) + "LE"]() to scan the block.
var block_align = data.data_block_size;
if (data.subchunk_id === "data" && block_align === 4) {
// start of audio subchunk + 4 bytes for the label ("data") + 4 bytes for size)
var data_offset = data.subchunk_byte + 4 + 4;
var block_length = block.length;
// Increment our offset by block_align, since we're analyzing on the basis of it.
for (data_offset; (data_offset + block_align) < block_length; data_offset = data_offset + block_align) {
if (block.readUInt32LE(data_offset) !== 0) {
log.message(log.INFO, "Storing the first " + data_offset + " bytes seperately");
// return the offset with first non-zero audio data;
return data_offset;
}
}
// if we didn't return out of the for loop, return default
return default_size;
} else {
// If we didn't find a data chunk, return default
return default_size;
}
};
function dasherize(s){
return s.replace(/_/g, '-');
}
module.exports.request_parameters = function request_parameters(accepted_params, uri, headers){
var u = url.parse(uri, true);
return accepted_params.reduce(function(o, p) {
o[p] = u.query[p] || headers["x-" + dasherize(p)];
return o;
}, {});
};
module.exports.target_from_url = function target_from_url(uri) {
var parsed = url.parse(uri);
var pathname = parsed.pathname;
if (pathname.substring(0,2) !== "/.") {
return "/" + parsed.hostname.split(".").reverse().join(".") + pathname;
} else {
return "/" + pathname.substring(2);
}
};

39
lib/validate.js

@ -0,0 +1,39 @@
"use strict";
/* globals require, module */
var log = require("../jlog.js");
var utils = require("./utils.js");
var time_valid = function time_valid(expires){
return !expires || ( expires >= new Date().getTime() );
};
var expected_token = function expected_token(inode, method, params) {
var expected_token = utils.sha1_to_hex(inode.access_key + method + (params.expires || ""));
log.message(log.DEBUG,"expected_token: " + expected_token);
log.message(log.DEBUG,"access_token: " + params.access_token);
return expected_token === params.access_token;
};
var token_valid = function token_valid(inode, method, params){
// don't bother validating tokens for HEAD, OPTIONS requests
// jjg - 08172015: might make sense to address this by removing the check from
// the method handlers below, but since I'm not sure if this is
// permanent, this is cleaner for now
return !!params.access_token &&
(( method === "HEAD" || method === "OPTIONS" ) ||
( time_valid(params.expires) && expected_token.apply(null, Array.prototype.slice.call(arguments)) ));
};
var has_key = function has_key(inode, params){
return params.access_key && params.access_key === inode.access_key;
};
module.exports.has_key = has_key;
module.exports.is_authorized = function is_authorized(inode, method, params){
return has_key(inode, params) ||
token_valid.apply(null, Array.prototype.slice.call(arguments));
};

47
package.json

@ -0,0 +1,47 @@
{
"name": "jsfs",
"version": "4.2.0",
"description": "deduplicating filesystem with a REST interface",
"main": "server.js",
"files": [
"server.js",
"config.ex",
"jlog.js",
"lib"
],
"directories": {
"lib": "./lib",
"tools": "./tools"
},
"repository": {
"type": "git",
"url": "https://github.com/jjg/jsfs.git"
},
"homepage": "https://github.com/jjg/jsfs",
"keywords": [
"filesystem",
"deduplicate",
"http",
"rest"
],
"author": {
"name": "Jason Gullickson"
},
"contributors": [
{
"name": "Marc Brakken"
}
],
"license": "GPL-3.0",
"scripts": {
"test": "./node_modules/.bin/mocha --reporter spec"
},
"dependencies": {
"through": "2.3.8"
},
"devDependencies": {
"chai": "^3.5.0",
"mocha": "^3.2.0",
"mock-fs": "^4.0.0-beta.1"
}
}

940
server.js

File diff suppressed because it is too large

65
test/file-types.js

@ -0,0 +1,65 @@
var fs = require("fs");
var expect = require('chai').expect
var file_types = require("../lib/file-types.js");
var config = require("../config.js");
var BLOCK_SIZE = config.BLOCK_SIZE;
var WAVE_RESULT = {
bitrate : 44100,
channels : 2,
data_block_size : 4,
duration : 302.26666666666665,
resolution : 16,
size : 53319876,
subchunk_byte : 36,
subchunk_id : "data",
type : "wave"
};
var UNKNOWN_RESULT = { type: "unknown" };
function load_test_block(file, callback) {
fs.readFile(file, function(err, data){
if (err) {
return callback(err);
}
return callback(null, data.slice(0, BLOCK_SIZE));
});
}
describe("file-types.js", function() {
describe("#analyze", function() {
it("should return full result for wave files", function(done) {
load_test_block("./test/fixtures/test.wav", function(error, block){
if (error) {
done(error);
} else {
var result = file_types.analyze(block);
expect(result).to.be.an("object");
expect(result).to.deep.equal(WAVE_RESULT);
expect(result).to.have.all.keys(Object.keys(WAVE_RESULT));
done();
}
});
});
it("should return `{ type: \"unknown\" }` for mp3 (anything not wave)", function(done) {
load_test_block("./test/fixtures/test.mp3", function(error, block){
if (error) {
done(error);
} else {
var result = file_types.analyze(block);
expect(result).to.be.an("object");
expect(result).to.deep.equal(UNKNOWN_RESULT);
expect(result).to.have.all.keys("type");
done();
}
});
});
});
});

BIN
test/fixtures/test.mp3

Binary file not shown.

BIN
test/fixtures/test.wav

Binary file not shown.

237
test/utils.js

@ -0,0 +1,237 @@
var fs = require("fs");
var mock = require('mock-fs');
var expect = require('chai').expect
var utils = require("../lib/utils.js");
var file_types = require("../lib/file-types.js");
var static = require("../lib/static.js");
var config = require("../config.js");
var log = require("../jlog.js");
var DEFAULT_STORAGE = config.STORAGE_LOCATIONS;
var BLOCK_SIZE = config.BLOCK_SIZE;
var TEST_PATH = "/com.jsfs.test/path/to/file.json";
var ACCEPTED_PARAMS = static.ACCEPTED_PARAMS;
function load_test_block(file, callback) {
fs.readFile(file, function(err, data){
if (err) {
return callback(err);
}
return callback(null, data.slice(0, BLOCK_SIZE));
});
}
describe("utils.js", function() {
before(function(){
// suppress debug log output for tests
log.level = 4;
});
after(function(){
// restore default log level
log.level = config.LOG_LEVEL
});
describe("#wave_audio_offset(block, data, default_size)", function() {
it("should return smaller offset for wave", function(done) {
load_test_block("./test/fixtures/test.wav", function(error, block){
if (error) {
done(error);
} else {
var offset = utils.wave_audio_offset(block, file_types.analyze(block), BLOCK_SIZE);
expect(offset).to.be.a("number");
expect(offset).to.equal(44);
expect(offset).to.be.at.most(BLOCK_SIZE);
expect(offset).to.be.below(BLOCK_SIZE);
done();
}
});
});
it("should return default offset for not wave", function(done) {
load_test_block("./test/fixtures/test.mp3", function(error, block){
if (error) {
done(error);
} else {
var offset = utils.wave_audio_offset(block, file_types.analyze(block), BLOCK_SIZE);
expect(offset).to.be.a("number");
expect(offset).to.equal(BLOCK_SIZE);
expect(offset).to.be.at.most(BLOCK_SIZE);
done();
}
});
});
});
context("inode operations", function(){
before(function(){
config.STORAGE_LOCATIONS = [
{"path":"fake/blocks1/","capacity":4294967296},
{"path":"fake/blocks2/","capacity":4294967296}
];
var fake_data = {
fake: {
"blocks1": {
},
"blocks2": {
}
}
};
var hash_1 = utils.sha1_to_hex("test_inode_1");
var hash_2 = utils.sha1_to_hex("test_inode_2");
fake_data.fake.blocks1[hash_1 + ".json"] = "{}";
fake_data.fake.blocks2[hash_1 + ".json"] = "{}";
fake_data.fake.blocks2[hash_2 + ".json"] = "{}";
mock(fake_data);
});
after(function(){
// restore default log level
mock.restore();
config.STORAGE_LOCATIONS = DEFAULT_STORAGE;
});
describe("#load_inode(url, callback)", function() {
it("should find an inode", function(done) {
utils.load_inode("test_inode_1", function(err, inode){
if (err) {
done(err);
} else {
done();
}
});
});
it("should searche multiple directories and return found inode", function(done) {
utils.load_inode("test_inode_2", function(err, inode){
if (err) {
done(err);
} else {
done();
}
});
});
it("should return an error for missing inode", function(done) {
utils.load_inode("test_inode_3", function(err, inode){
expect(inode).to.be.undefined;
expect(err).to.be.an.instanceof(Error);
expect(err).to.have.property("code", "ENOENT");
expect(err).to.have.property("errno", 34);
done();
});
});
});
describe("#save_inode(inode, callback)", function(){
it("should save an inode", function(done) {
var path = "test_inode_4";
var inode = { fingerprint: utils.sha1_to_hex(path) };
utils.save_inode(inode, function(found_inode){
expect(found_inode).to.deep.equal(inode);
// maybe should do manual inspection of each directory
utils.load_inode(path, function(err, response){
expect(err).to.be.null;
expect(response).to.be.an("object");
expect(response).to.deep.equal(inode);
done();
});
});
});
});
});
describe("#target_from_url(uri)", function() {
it("should set target from url", function() {
var test_uri = "http://test.jsfs.com/path/to/file.json";
var result = utils.target_from_url(test_uri);
expect(result).to.be.a("string");
expect(result).to.equal(TEST_PATH);
});
it("should return fully specificed target path", function() {
var test_uri = "http://test2.jsfs.com/.com.jsfs.test/path/to/file.json";
var result = utils.target_from_url(test_uri);
expect(result).to.be.a("string");
expect(result).to.equal(TEST_PATH);
});
it("should ignore the port", function() {
var test_uri = "http://test.jsfs.com:1234/path/to/file.json";
var result = utils.target_from_url(test_uri);
expect(result).to.be.a("string");
expect(result).to.equal(TEST_PATH);
});
it("should ignore query params", function() {
var test_uri = "http://test.jsfs.com/path/to/file.json?test=query&more=fun";
var result = utils.target_from_url(test_uri);
expect(result).to.be.a("string");
expect(result).to.equal(TEST_PATH);
});
});
describe("#request_parameters", function() {
it("should return object with all parameters", function() {
var test_uri = "http://test.jsfs.com/path/to/file.json";
var headers = {};
var result = utils.request_parameters(ACCEPTED_PARAMS, test_uri, headers);
expect(result).to.be.an("object");
expect(Object.keys(result)).to.have.lengthOf(ACCEPTED_PARAMS.length);
expect(result).to.have.all.keys(ACCEPTED_PARAMS);
});
it("should set parameters from query", function() {
var test_uri = "http://test.jsfs.com/path/to/file.json?access_token=testing";
var headers = {};
var result = utils.request_parameters(ACCEPTED_PARAMS, test_uri, headers);
expect(result).to.be.an("object");
expect(result.access_token).to.equal("testing");
});
it("should set parameters from header", function() {
var test_uri = "http://test.jsfs.com/path/to/file.json";
var headers = {"x-access-token": "testing"};
var result = utils.request_parameters(ACCEPTED_PARAMS, test_uri, headers);
expect(result).to.be.an("object");
expect(result.access_token).to.equal("testing");
});
it("should give priority to url query over header", function() {
var test_uri = "http://test.jsfs.com/path/to/file.json?access_token=testing";
var headers = {"x-access-token": "ignore_me"};
var result = utils.request_parameters(ACCEPTED_PARAMS, test_uri, headers);
expect(result).to.be.an("object");
expect(result.access_token).to.equal("testing");
});
});
});

121
test/validate.js

@ -0,0 +1,121 @@
var expect = require('chai').expect
var validate = require("../lib/validate.js");
var log = require("../jlog.js");
var config = require("../config.js");
var utils = require("../lib/utils.js");
var GOOD_KEY = "testing_key";
var BAD_KEY = "wrong_key";
var INODE = { access_key: GOOD_KEY };
var GET = "GET";
function setExpire(minutes){
var d = new Date();
d.setMinutes(d.getMinutes() + minutes);
return d.getTime();
}
describe("validation.js", function(){
before(function(){
// suppress debug log output for tests
log.level = 4;
});
after(function(){
// restore default log level
log.level = config.LOG_LEVEL
});
describe("#has_key(inode, params)", function() {
it("should validate an access_key", function() {
var params = { access_key: GOOD_KEY };
var result = validate.has_key(INODE, params);
expect(result).to.be.true;
});
it("should reject an incorrect access_key", function() {
var params = { access_key: BAD_KEY };
var result = validate.has_key(INODE, params);
expect(result).to.be.false;
});
});
describe("#is_authorized(inode, method, params)", function() {
it("should validate an access_key", function() {
var params = { access_key: GOOD_KEY };
var result = validate.is_authorized(INODE, GET, params);
expect(result).to.be.true;
});
it("should reject an incorrect access key", function(){
var params = { access_key: BAD_KEY };
var result = validate.is_authorized(INODE, GET, params);
expect(result).to.be.false;
});
it("should validate an access token", function(){
var params = { access_token: utils.sha1_to_hex(GOOD_KEY + GET) };
var result = validate.is_authorized(INODE, GET, params);
expect(result).to.be.true;
});
it("should reject an access token for wrong method", function(){
var params = { access_token: utils.sha1_to_hex(GOOD_KEY + "POST") };
var result = validate.is_authorized(INODE, GET, params);
expect(result).to.be.false;
});
it("should reject wrong access token", function(){
var params = { access_token: utils.sha1_to_hex(BAD_KEY + GET) };
var result = validate.is_authorized(INODE, GET, params);
expect(result).to.be.false;
});
it("should validate a future time token", function() {
var expires = setExpire(30);
var params = {
access_token : utils.sha1_to_hex(GOOD_KEY + GET + expires),
expires : expires
};
var result = validate.is_authorized(INODE, GET, params)
expect(result).to.be.true;
});
it("should reject an expired time token", function() {
var expires = setExpire(-1);
var params = {
access_token : utils.sha1_to_hex(GOOD_KEY + GET + expires),
expires : expires
};
var result = validate.is_authorized(INODE, GET, params);
expect(result).to.be.false;
});
it("should validate HEAD requests", function(){
var params = { access_token: utils.sha1_to_hex(BAD_KEY + GET) };
var result = validate.is_authorized(INODE, "HEAD", params);
expect(result).to.be.true;
});
it("should validate OPTIONS requests", function(){
var params = { access_token: utils.sha1_to_hex(BAD_KEY + GET) };
var result = validate.is_authorized(INODE, "OPTIONS", params);
expect(result).to.be.true;
});
});
});

64
tools/compress_blocks.js

@ -1,24 +1,58 @@
var fs = require("fs");
var zlib = require("zlib");
var files;
var block_location;
var total;
var process_next_file = function process_next_file(){
console.log((((total - files.length) / total) * 100) + "% completed");
if (files.length > 0) {
process_file(files.pop());
} else {
console.log("No more blocks to compress");
}
};
var process_file = function process_file(file){
var selected_file = block_location + "/" + file;
if(selected_file.indexOf(".gz") < 0 && selected_file.indexOf(".json") < 0){
var on_finish = function on_finish(){
writer.removeListener("finish", on_finish);
fs.unlink(selected_file, function(err){
if (err) {
console.log(err);
return;
}
process_next_file();
});
};
var writer = fs.createWriteStream(selected_file + ".gz");
writer.on("finish", on_finish);
fs.createReadStream(selected_file).pipe(zlib.createGzip()).pipe(writer);
} else {
process_next_file();
}
};
if(process.argv[2]){
var block_location = process.argv[2]
var files = fs.readdirSync(block_location);
for(file in files){
var selected_file = block_location + "/" + files[file];
if(selected_file.indexOf(".gz") < 0 && selected_file.indexOf(".json") < 0){
console.log(Math.round((file / files.length * 100)) + "% complete");
fs.writeFileSync(selected_file + ".gz", zlib.gzipSync(fs.readFileSync(selected_file)));
// delete uncompressed block
fs.unlink(selected_file);
block_location = process.argv[2]
fs.readdir(block_location, function(err, _files){
if (err) {
console.log(err);
return;
}
}
console.log("No more blocks to compress");
files = _files;
total = files.length;
process_next_file();
});
} else {
console.log("usage: compress_blocks.js <path to blocks>");
}

Loading…
Cancel
Save