diff --git a/gulpfile.js b/gulpfile.js index 26a03eb4190d66f6f0d3e3458ebe0bc4032b9a9a..2b759ea39d98e6abd8fef07ebb71cb0a5f31d27a 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -71,12 +71,13 @@ var polyfills = [ var concatOrder = [ 'y.js', 'Connector.js', - 'OperationStore.js', + 'Database.js', + 'Transaction.js', 'Struct.js', 'Utils.js', - 'OperationStores/RedBlackTree.js', - 'OperationStores/Memory.js', - 'OperationStores/IndexedDB.js', + 'Databases/RedBlackTree.js', + 'Databases/Memory.js', + 'Databases/IndexedDB.js', 'Connectors/Test.js', 'Connectors/WebRTC.js', 'Types/Array.js', diff --git a/src/Database.js b/src/Database.js new file mode 100644 index 0000000000000000000000000000000000000000..c8e5fcd28a9f4b9936e484c013ff7d4916afa039 --- /dev/null +++ b/src/Database.js @@ -0,0 +1,283 @@ +/* global Y */ +'use strict' + +/* + Partial definition of an OperationStore. + TODO: name it Database, operation store only holds operations. + + A database definition must alse define the following methods: + * logTable() (optional) + - show relevant information information in a table + * requestTransaction(makeGen) + - request a transaction + * destroy() + - destroy the database +*/ +class AbstractOperationStore { + constructor (y, opts) { + this.y = y + // E.g. this.listenersById[id] : Array<Listener> + this.listenersById = {} + // Execute the next time a transaction is requested + this.listenersByIdExecuteNow = [] + // A transaction is requested + this.listenersByIdRequestPending = false + /* To make things more clear, the following naming conventions: + * ls : we put this.listenersById on ls + * l : Array<Listener> + * id : Id (can't use as property name) + * sid : String (converted from id via JSON.stringify + so we can use it as a property name) + + Always remember to first overwrite + a property before you iterate over it! + */ + // TODO: Use ES7 Weak Maps. This way types that are no longer user, + // wont be kept in memory. + this.initializedTypes = {} + this.whenUserIdSetListener = null + + this.gc1 = [] // first stage + this.gc2 = [] // second stage -> after that, remove the op + this.gcTimeout = opts.gcTimeout || 5000 + var os = this + function garbageCollect () { + return new Promise((resolve) => { + os.requestTransaction(function * () { + if (os.y.connector.isSynced) { + for (var i in os.gc2) { + var oid = os.gc2[i] + yield* this.garbageCollectOperation(oid) + } + os.gc2 = os.gc1 + os.gc1 = [] + } + if (os.gcTimeout > 0) { + os.gcInterval = setTimeout(garbageCollect, os.gcTimeout) + } + resolve() + }) + }) + } + this.garbageCollect = garbageCollect + if (this.gcTimeout > 0) { + garbageCollect() + } + } + stopGarbageCollector () { + var self = this + return new Promise(function (resolve) { + self.requestTransaction(function * () { + var ungc = self.gc1.concat(self.gc2) + self.gc1 = [] + self.gc2 = [] + for (var i in ungc) { + var op = yield* this.getOperation(ungc[i]) + delete op.gc + yield* this.setOperation(op) + } + resolve() + }) + }) + } + * garbageCollectAfterSync () { + this.requestTransaction(function * () { + yield* this.os.iterate(this, null, null, function * (op) { + if (op.deleted && op.left != null) { + var left = yield this.os.find(op.left) + this.store.addToGarbageCollector(op, left) + } + }) + }) + } + /* + Try to add to GC. + + TODO: rename this function + + Rulez: + * Only gc if this user is online + * The most left element in a list must not be gc'd. + => There is at least one element in the list + + returns true iff op was added to GC + */ + addToGarbageCollector (op, left) { + if ( + op.gc == null && + op.deleted === true && + this.y.connector.isSynced && + left != null && + left.deleted === true + ) { + op.gc = true + this.gc1.push(op.id) + return true + } else { + return false + } + } + removeFromGarbageCollector (op) { + function filter (o) { + return !Y.utils.compareIds(o, op.id) + } + this.gc1 = this.gc1.filter(filter) + this.gc2 = this.gc2.filter(filter) + delete op.gc + } + destroy () { + clearInterval(this.gcInterval) + this.gcInterval = null + } + setUserId (userId) { + this.userId = userId + this.opClock = 0 + if (this.whenUserIdSetListener != null) { + this.whenUserIdSetListener() + this.whenUserIdSetListener = null + } + } + whenUserIdSet (f) { + if (this.userId != null) { + f() + } else { + this.whenUserIdSetListener = f + } + } + getNextOpId () { + if (this.userId == null) { + throw new Error('OperationStore not yet initialized!') + } + return [this.userId, this.opClock++] + } + /* + Apply a list of operations. + + * get a transaction + * check whether all Struct.*.requiredOps are in the OS + * check if it is an expected op (otherwise wait for it) + * check if was deleted, apply a delete operation after op was applied + */ + apply (ops) { + for (var key in ops) { + var o = ops[key] + var required = Y.Struct[o.struct].requiredOps(o) + this.whenOperationsExist(required, o) + } + } + /* + op is executed as soon as every operation requested is available. + Note that Transaction can (and should) buffer requests. + */ + whenOperationsExist (ids, op) { + if (ids.length > 0) { + let listener = { + op: op, + missing: ids.length + } + + for (let key in ids) { + let id = ids[key] + let sid = JSON.stringify(id) + let l = this.listenersById[sid] + if (l == null) { + l = [] + this.listenersById[sid] = l + } + l.push(listener) + } + } else { + this.listenersByIdExecuteNow.push({ + op: op + }) + } + + if (this.listenersByIdRequestPending) { + return + } + + this.listenersByIdRequestPending = true + var store = this + + this.requestTransaction(function * () { + var exeNow = store.listenersByIdExecuteNow + store.listenersByIdExecuteNow = [] + + var ls = store.listenersById + store.listenersById = {} + + store.listenersByIdRequestPending = false + + for (let key in exeNow) { + let o = exeNow[key].op + yield* store.tryExecute.call(this, o) + } + + for (var sid in ls) { + var l = ls[sid] + var id = JSON.parse(sid) + if ((yield* this.getOperation(id)) == null) { + store.listenersById[sid] = l + } else { + for (let key in l) { + let listener = l[key] + let o = listener.op + if (--listener.missing === 0) { + yield* store.tryExecute.call(this, o) + } + } + } + } + }) + } + /* + Actually execute an operation, when all expected operations are available. + */ + * tryExecute (op) { + if (op.struct === 'Delete') { + yield* Y.Struct.Delete.execute.call(this, op) + } else if ((yield* this.getOperation(op.id)) == null && !(yield* this.isGarbageCollected(op.id))) { + yield* Y.Struct[op.struct].execute.call(this, op) + var next = yield* this.addOperation(op) + yield* this.store.operationAdded(this, op, next) + + // Delete if DS says this is actually deleted + if (yield* this.isDeleted(op.id)) { + yield* Y.Struct['Delete'].execute.call(this, {struct: 'Delete', target: op.id}) + } + } + } + // called by a transaction when an operation is added + * operationAdded (transaction, op, next) { + // increase SS + var o = op + var state = yield* transaction.getState(op.id[0]) + while (o != null && o.id[1] === state.clock && op.id[0] === o.id[0]) { + // either its a new operation (1. case), or it is an operation that was deleted, but is not yet in the OS + state.clock++ + yield* transaction.checkDeleteStoreForState(state) + o = next() + } + yield* transaction.setState(state) + + // notify whenOperation listeners (by id) + var sid = JSON.stringify(op.id) + var l = this.listenersById[sid] + delete this.listenersById[sid] + + if (l != null) { + for (var key in l) { + var listener = l[key] + if (--listener.missing === 0) { + this.whenOperationsExist([], listener.op) + } + } + } + // notify parent, if it has been initialized as a custom type + var t = this.initializedTypes[JSON.stringify(op.parent)] + if (t != null && !op.deleted) { + yield* t._changed(transaction, Y.utils.copyObject(op)) + } + } +} +Y.AbstractOperationStore = AbstractOperationStore diff --git a/src/Database.spec.js b/src/Database.spec.js new file mode 100644 index 0000000000000000000000000000000000000000..5de2dc9866482046b68a39582caae0035bba3526 --- /dev/null +++ b/src/Database.spec.js @@ -0,0 +1,329 @@ +/* global Y, async */ +/* eslint-env browser,jasmine,console */ +var databases = ['Memory'] +for (var database of databases) { + describe(`Database (${database})`, function () { + var store + describe('DeleteStore', function () { + describe('Basic', function () { + beforeEach(function () { + store = new Y[database](null, { + gcTimeout: -1 + }) + }) + it('Deleted operation is deleted', async(function * (done) { + store.requestTransaction(function * () { + yield* this.markDeleted(['u1', 10]) + expect(yield* this.isDeleted(['u1', 10])).toBeTruthy() + expect(yield* this.getDeleteSet()).toEqual({'u1': [[10, 1, false]]}) + done() + }) + })) + it('Deleted operation extends other deleted operation', async(function * (done) { + store.requestTransaction(function * () { + yield* this.markDeleted(['u1', 10]) + yield* this.markDeleted(['u1', 11]) + expect(yield* this.isDeleted(['u1', 10])).toBeTruthy() + expect(yield* this.isDeleted(['u1', 11])).toBeTruthy() + expect(yield* this.getDeleteSet()).toEqual({'u1': [[10, 2, false]]}) + done() + }) + })) + it('Deleted operation extends other deleted operation', async(function * (done) { + store.requestTransaction(function * () { + yield* this.markDeleted(['0', 3]) + yield* this.markDeleted(['0', 4]) + yield* this.markDeleted(['0', 2]) + expect(yield* this.getDeleteSet()).toEqual({'0': [[2, 3, false]]}) + done() + }) + })) + it('Debug #1', async(function * (done) { + store.requestTransaction(function * () { + yield* this.markDeleted(['166', 0]) + yield* this.markDeleted(['166', 2]) + yield* this.markDeleted(['166', 0]) + yield* this.markDeleted(['166', 2]) + yield* this.markGarbageCollected(['166', 2]) + yield* this.markDeleted(['166', 1]) + yield* this.markDeleted(['166', 3]) + yield* this.markGarbageCollected(['166', 3]) + yield* this.markDeleted(['166', 0]) + expect(yield* this.getDeleteSet()).toEqual({'166': [[0, 2, false], [2, 2, true]]}) + done() + }) + })) + it('Debug #2', async(function * (done) { + store.requestTransaction(function * () { + yield* this.markDeleted(['293', 0]) + yield* this.markDeleted(['291', 2]) + yield* this.markDeleted(['291', 2]) + yield* this.markGarbageCollected(['293', 0]) + yield* this.markDeleted(['293', 1]) + yield* this.markGarbageCollected(['291', 2]) + expect(yield* this.getDeleteSet()).toEqual({'291': [[2, 1, true]], '293': [[0, 1, true], [1, 1, false]]}) + done() + }) + })) + it('Debug #3', async(function * (done) { + store.requestTransaction(function * () { + yield* this.markDeleted(['581', 0]) + yield* this.markDeleted(['581', 1]) + yield* this.markDeleted(['580', 0]) + yield* this.markDeleted(['580', 0]) + yield* this.markGarbageCollected(['581', 0]) + yield* this.markDeleted(['581', 2]) + yield* this.markDeleted(['580', 1]) + yield* this.markDeleted(['580', 2]) + yield* this.markDeleted(['580', 1]) + yield* this.markDeleted(['580', 2]) + yield* this.markGarbageCollected(['581', 2]) + yield* this.markGarbageCollected(['581', 1]) + yield* this.markGarbageCollected(['580', 1]) + expect(yield* this.getDeleteSet()).toEqual({'580': [[0, 1, false], [1, 1, true], [2, 1, false]], '581': [[0, 3, true]]}) + done() + }) + })) + it('Debug #4', async(function * (done) { + store.requestTransaction(function * () { + yield* this.markDeleted(['544', 0]) + yield* this.markDeleted(['543', 2]) + yield* this.markDeleted(['544', 0]) + yield* this.markDeleted(['543', 2]) + yield* this.markGarbageCollected(['544', 0]) + yield* this.markDeleted(['545', 1]) + yield* this.markDeleted(['543', 4]) + yield* this.markDeleted(['543', 3]) + yield* this.markDeleted(['544', 1]) + yield* this.markDeleted(['544', 2]) + yield* this.markDeleted(['544', 1]) + yield* this.markDeleted(['544', 2]) + yield* this.markGarbageCollected(['543', 2]) + yield* this.markGarbageCollected(['543', 4]) + yield* this.markGarbageCollected(['544', 2]) + yield* this.markGarbageCollected(['543', 3]) + expect(yield* this.getDeleteSet()).toEqual({'543': [[2, 3, true]], '544': [[0, 1, true], [1, 1, false], [2, 1, true]], '545': [[1, 1, false]]}) + done() + }) + })) + it('Debug #5', async(function * (done) { + store.requestTransaction(function * () { + yield* this.applyDeleteSet({'16': [[1, 2, false]], '17': [[0, 1, true], [1, 3, false]]}) + expect(yield* this.getDeleteSet()).toEqual({'16': [[1, 2, false]], '17': [[0, 1, true], [1, 3, false]]}) + yield* this.applyDeleteSet({'16': [[1, 2, false]], '17': [[0, 4, true]]}) + expect(yield* this.getDeleteSet()).toEqual({'16': [[1, 2, false]], '17': [[0, 4, true]]}) + done() + }) + })) + it('Debug #6', async(function * (done) { + store.requestTransaction(function * () { + yield* this.applyDeleteSet({'40': [[0, 3, false]]}) + expect(yield* this.getDeleteSet()).toEqual({'40': [[0, 3, false]]}) + yield* this.applyDeleteSet({'39': [[2, 2, false]], '40': [[0, 1, true], [1, 2, false]], '41': [[2, 1, false]]}) + expect(yield* this.getDeleteSet()).toEqual({'39': [[2, 2, false]], '40': [[0, 1, true], [1, 2, false]], '41': [[2, 1, false]]}) + done() + }) + })) + it('Debug #7', async(function * (done) { + store.requestTransaction(function * () { + yield* this.markDeleted(['9', 2]) + yield* this.markDeleted(['11', 2]) + yield* this.markDeleted(['11', 4]) + yield* this.markDeleted(['11', 1]) + yield* this.markDeleted(['9', 4]) + yield* this.markDeleted(['10', 0]) + yield* this.markGarbageCollected(['11', 2]) + yield* this.markDeleted(['11', 2]) + yield* this.markGarbageCollected(['11', 3]) + yield* this.markDeleted(['11', 3]) + yield* this.markDeleted(['11', 3]) + yield* this.markDeleted(['9', 4]) + yield* this.markDeleted(['10', 0]) + yield* this.markGarbageCollected(['11', 1]) + yield* this.markDeleted(['11', 1]) + expect(yield* this.getDeleteSet()).toEqual({'9': [[2, 1, false], [4, 1, false]], '10': [[0, 1, false]], '11': [[1, 3, true], [4, 1, false]]}) + done() + }) + })) + }) + }) + describe('OperationStore', function () { + describe('Basic Tests', function () { + beforeEach(function () { + store = new Y[database](null, { + gcTimeout: -1 + }) + }) + it('debug #1', function (done) { + store.requestTransaction(function * () { + yield this.os.set({id: [2]}) + yield this.os.set({id: [0]}) + yield this.os.delete([2]) + yield this.os.set({id: [1]}) + expect(yield this.os.find([0])).not.toBeNull() + expect(yield this.os.find([1])).not.toBeNull() + expect(yield this.os.find([2])).toBeNull() + done() + }) + }) + it('can add&retrieve 5 elements', function (done) { + store.requestTransaction(function * () { + yield this.os.set({val: 'four', id: [4]}) + yield this.os.set({val: 'one', id: [1]}) + yield this.os.set({val: 'three', id: [3]}) + yield this.os.set({val: 'two', id: [2]}) + yield this.os.set({val: 'five', id: [5]}) + expect((yield this.os.find([1])).val).toEqual('one') + expect((yield this.os.find([2])).val).toEqual('two') + expect((yield this.os.find([3])).val).toEqual('three') + expect((yield this.os.find([4])).val).toEqual('four') + expect((yield this.os.find([5])).val).toEqual('five') + done() + }) + }) + it('5 elements do not exist anymore after deleting them', function (done) { + store.requestTransaction(function * () { + yield this.os.set({val: 'four', id: [4]}) + yield this.os.set({val: 'one', id: [1]}) + yield this.os.set({val: 'three', id: [3]}) + yield this.os.set({val: 'two', id: [2]}) + yield this.os.set({val: 'five', id: [5]}) + yield this.os.delete([4]) + expect(yield this.os.find([4])).not.toBeTruthy() + yield this.os.delete([3]) + expect(yield this.os.find([3])).not.toBeTruthy() + yield this.os.delete([2]) + expect(yield this.os.find([2])).not.toBeTruthy() + yield this.os.delete([1]) + expect(yield this.os.find([1])).not.toBeTruthy() + yield this.os.delete([5]) + expect(yield this.os.find([5])).not.toBeTruthy() + done() + }) + }) + }) + var numberOfOSTests = 1000 + describe(`Random Tests - after adding&deleting (0.8/0.2) ${numberOfOSTests} times`, function () { + var elements = [] + beforeAll(function (done) { + store = new Y[database](null, { + gcTimeout: -1 + }) + store.requestTransaction(function * () { + for (var i = 0; i < numberOfOSTests; i++) { + var r = Math.random() + if (r < 0.8) { + var obj = [Math.floor(Math.random() * numberOfOSTests * 10000)] + if (!(yield this.os.findNode(obj))) { + elements.push(obj) + yield this.os.set({id: obj}) + } + } else if (elements.length > 0) { + var elemid = Math.floor(Math.random() * elements.length) + var elem = elements[elemid] + elements = elements.filter(function (e) { + return !Y.utils.compareIds(e, elem) + }) + yield this.os.delete(elem) + } + } + done() + }) + }) + it('can find every object', function (done) { + store.requestTransaction(function * () { + for (var id of elements) { + expect((yield this.os.find(id)).id).toEqual(id) + } + done() + }) + }) + + it('can find every object with lower bound search', function (done) { + store.requestTransaction(function * () { + for (var id of elements) { + expect((yield this.os.findNodeWithLowerBound(id)).val.id).toEqual(id) + } + done() + }) + }) + + it('iterating over a tree with lower bound yields the right amount of results', function (done) { + var lowerBound = elements[Math.floor(Math.random() * elements.length)] + var expectedResults = elements.filter(function (e, pos) { + return (Y.utils.smaller(lowerBound, e) || Y.utils.compareIds(e, lowerBound)) && elements.indexOf(e) === pos + }).length + + var actualResults = 0 + store.requestTransaction(function * () { + yield* this.os.iterate(this, lowerBound, null, function * (val) { + expect(val).toBeDefined() + actualResults++ + }) + expect(expectedResults).toEqual(actualResults) + done() + }) + }) + + it('iterating over a tree without bounds yield the right amount of results', function (done) { + var lowerBound = null + var expectedResults = elements.filter(function (e, pos) { + return elements.indexOf(e) === pos + }).length + var actualResults = 0 + store.requestTransaction(function * () { + yield* this.os.iterate(this, lowerBound, null, function * (val) { + expect(val).toBeDefined() + actualResults++ + }) + expect(expectedResults).toEqual(actualResults) + done() + }) + }) + + it('iterating over a tree with upper bound yields the right amount of results', function (done) { + var upperBound = elements[Math.floor(Math.random() * elements.length)] + var expectedResults = elements.filter(function (e, pos) { + return (Y.utils.smaller(e, upperBound) || Y.utils.compareIds(e, upperBound)) && elements.indexOf(e) === pos + }).length + + var actualResults = 0 + store.requestTransaction(function * () { + yield* this.os.iterate(this, null, upperBound, function * (val) { + expect(val).toBeDefined() + actualResults++ + }) + expect(expectedResults).toEqual(actualResults) + done() + }) + }) + + it('iterating over a tree with upper and lower bounds yield the right amount of results', function (done) { + var b1 = elements[Math.floor(Math.random() * elements.length)] + var b2 = elements[Math.floor(Math.random() * elements.length)] + var upperBound, lowerBound + if (Y.utils.smaller(b1, b2)) { + lowerBound = b1 + upperBound = b2 + } else { + lowerBound = b2 + upperBound = b1 + } + var expectedResults = elements.filter(function (e, pos) { + return (Y.utils.smaller(lowerBound, e) || Y.utils.compareIds(e, lowerBound)) && + (Y.utils.smaller(e, upperBound) || Y.utils.compareIds(e, upperBound)) && elements.indexOf(e) === pos + }).length + var actualResults = 0 + store.requestTransaction(function * () { + yield* this.os.iterate(this, lowerBound, upperBound, function * (val) { + expect(val).toBeDefined() + actualResults++ + }) + expect(expectedResults).toEqual(actualResults) + done() + }) + }) + }) + }) + }) +} diff --git a/src/OperationStores/IndexedDB.js b/src/Databases/IndexedDB.js similarity index 95% rename from src/OperationStores/IndexedDB.js rename to src/Databases/IndexedDB.js index ca36f678ec746a2a1c434736bdd4728e0e9fb0c3..89391d7250e318b5dc33ed4067937a6fcb3270bf 100644 --- a/src/OperationStores/IndexedDB.js +++ b/src/Databases/IndexedDB.js @@ -1,7 +1,9 @@ +/* global Y */ + 'use strict' -Y.IndexedDB = (function () { // eslint-disable-line - class Transaction extends Y.AbstractTransaction { // eslint-disable-line +Y.IndexedDB = (function () { + class Transaction extends Y.AbstractTransaction { constructor (store) { super(store) this.transaction = store.db.transaction(['OperationStore', 'StateVector'], 'readwrite') @@ -81,7 +83,7 @@ Y.IndexedDB = (function () { // eslint-disable-line return ops } } - class OperationStore extends Y.AbstractOperationStore { // eslint-disable-line no-undef + class OperationStore extends Y.AbstractOperationStore { constructor (y, opts) { super(y, opts) if (opts == null) { diff --git a/src/OperationStores/IndexedDB.spec.js b/src/Databases/IndexedDB.spec.js similarity index 95% rename from src/OperationStores/IndexedDB.spec.js rename to src/Databases/IndexedDB.spec.js index ae84fe6357cc05a0f6687529b6adf16dfeb195fe..6741894cab90acda722103f77f68d17ee60c8632 100644 --- a/src/OperationStores/IndexedDB.spec.js +++ b/src/Databases/IndexedDB.spec.js @@ -30,7 +30,7 @@ if (typeof window !== 'undefined' && false) { .toEqual(op) yield* this.removeOperation(['1', 0]) expect(yield* this.getOperation(['1', 0])) - .toBeUndefined() + .toBeNull() done() }) }) @@ -38,7 +38,7 @@ if (typeof window !== 'undefined' && false) { it('getOperation(op) returns undefined if op does not exist', function (done) { ob.requestTransaction(function *() { var op = yield* this.getOperation("plzDon'tBeThere") - expect(op).toBeUndefined() + expect(op).toBeNull() done() }) }) @@ -64,7 +64,6 @@ if (typeof window !== 'undefined' && false) { yield* this.setState(s1) yield* this.setState(s2) var sv = yield* this.getStateVector() - expect(sv).not.toBeUndefined() expect(sv).toEqual([s1, s2]) done() }) @@ -77,7 +76,6 @@ if (typeof window !== 'undefined' && false) { yield* this.setState(s1) yield* this.setState(s2) var sv = yield* this.getStateSet() - expect(sv).not.toBeUndefined() expect(sv).toEqual({ '1': 1, '2': 3 diff --git a/src/OperationStores/Memory.js b/src/Databases/Memory.js similarity index 96% rename from src/OperationStores/Memory.js rename to src/Databases/Memory.js index 23bbe188a8112e8d33c38f3e12a10be9c72cbd70..2cb3d670dc076dfbf29c3ccdf7277f3dd6cb142a 100644 --- a/src/OperationStores/Memory.js +++ b/src/Databases/Memory.js @@ -17,10 +17,10 @@ Y.Memory = (function () { constructor (y, opts) { super(y, opts) this.os = new Y.utils.RBTree() - this.ss = {} + this.ds = new Y.utils.RBTree() + this.ss = new Y.utils.RBTree() this.waitingTransactions = [] this.transactionInProgress = false - this.ds = new DeleteStore() } logTable () { var self = this diff --git a/src/OperationStores/RedBlackTree.js b/src/Databases/RedBlackTree.js similarity index 99% rename from src/OperationStores/RedBlackTree.js rename to src/Databases/RedBlackTree.js index 37201132d81be37bea4138fa4581289cc96a1847..a4a7211e804f34562a812e1772c85625051d3907 100644 --- a/src/OperationStores/RedBlackTree.js +++ b/src/Databases/RedBlackTree.js @@ -221,7 +221,8 @@ class RBTree { } } find (id) { - return this.findNode(id).val + var n + return (n = this.findNode(id)) ? n.val : null } findNode (id) { if (id == null || id.constructor !== Array) { @@ -387,7 +388,7 @@ class RBTree { } } } - add (v) { + set (v) { if (v == null || v.id == null || v.id.constructor !== Array) { throw new Error('v is expected to have an id property which is an Array!') } @@ -410,7 +411,8 @@ class RBTree { p = p.right } } else { - return null + p.val = node.val + return p } } this._fixInsert(node) diff --git a/src/OperationStores/RedBlackTree.spec.js b/src/Databases/RedBlackTree.spec.js similarity index 72% rename from src/OperationStores/RedBlackTree.spec.js rename to src/Databases/RedBlackTree.spec.js index e02a4af083ad52f42d416b7744417ac773f7b3f4..c8c6a427a7cbef0607bfbba213521504ae1c5394 100644 --- a/src/OperationStores/RedBlackTree.spec.js +++ b/src/Databases/RedBlackTree.spec.js @@ -57,57 +57,17 @@ describe('RedBlack Tree', function () { }) this.tree = this.memory.os }) - it('can add&retrieve 5 elements', function () { - this.tree.add({val: 'four', id: [4]}) - this.tree.add({val: 'one', id: [1]}) - this.tree.add({val: 'three', id: [3]}) - this.tree.add({val: 'two', id: [2]}) - this.tree.add({val: 'five', id: [5]}) - expect(this.tree.find([1]).val).toEqual('one') - expect(this.tree.find([2]).val).toEqual('two') - expect(this.tree.find([3]).val).toEqual('three') - expect(this.tree.find([4]).val).toEqual('four') - expect(this.tree.find([5]).val).toEqual('five') - }) - - it('5 elements do not exist anymore after deleting them', function () { - this.tree.add({val: 'four', id: [4]}) - this.tree.add({val: 'one', id: [1]}) - this.tree.add({val: 'three', id: [3]}) - this.tree.add({val: 'two', id: [2]}) - this.tree.add({val: 'five', id: [5]}) - this.tree.delete([4]) - expect(this.tree.find([4])).not.toBeTruthy() - this.tree.delete([3]) - expect(this.tree.find([3])).not.toBeTruthy() - this.tree.delete([2]) - expect(this.tree.find([2])).not.toBeTruthy() - this.tree.delete([1]) - expect(this.tree.find([1])).not.toBeTruthy() - this.tree.delete([5]) - expect(this.tree.find([5])).not.toBeTruthy() - }) - - it('debug #1', function () { - this.tree.add({id: [2]}) - this.tree.add({id: [0]}) - this.tree.delete([2]) - this.tree.add({id: [1]}) - expect(this.tree.find([0])).not.toBeUndefined() - expect(this.tree.find([1])).not.toBeUndefined() - expect(this.tree.find([2])).toBeUndefined() - }) describe('debug #2', function () { var tree = new Y.utils.RBTree() - tree.add({id: [8433]}) - tree.add({id: [12844]}) - tree.add({id: [1795]}) - tree.add({id: [30302]}) - tree.add({id: [64287]}) + tree.set({id: [8433]}) + tree.set({id: [12844]}) + tree.set({id: [1795]}) + tree.set({id: [30302]}) + tree.set({id: [64287]}) tree.delete([8433]) - tree.add({id: [28996]}) + tree.set({id: [28996]}) tree.delete([64287]) - tree.add({id: [22721]}) + tree.set({id: [22721]}) itRootNodeIsBlack(tree, []) itBlackHeightOfSubTreesAreEqual(tree, []) @@ -122,12 +82,14 @@ describe('RedBlack Tree', function () { var obj = [Math.floor(Math.random() * numberOfRBTreeTests * 10000)] if (!tree.findNode(obj)) { elements.push(obj) - tree.add({id: obj}) + tree.set({id: obj}) } } else if (elements.length > 0) { var elemid = Math.floor(Math.random() * elements.length) var elem = elements[elemid] - elements = elements.filter(function (e) {return !Y.utils.compareIds(e, elem); }); // eslint-disable-line + elements = elements.filter(function (e) { + return !Y.utils.compareIds(e, elem) + }) tree.delete(elem) } } @@ -157,7 +119,7 @@ describe('RedBlack Tree', function () { var actualResults = 0 this.memory.requestTransaction(function * () { yield* tree.iterate(this, lowerBound, null, function * (val) { - expect(val).not.toBeUndefined() + expect(val).toBeDefined() actualResults++ }) expect(expectedResults).toEqual(actualResults) @@ -173,7 +135,7 @@ describe('RedBlack Tree', function () { var actualResults = 0 this.memory.requestTransaction(function * () { yield* tree.iterate(this, lowerBound, null, function * (val) { - expect(val).not.toBeUndefined() + expect(val).toBeDefined() actualResults++ }) expect(expectedResults).toEqual(actualResults) @@ -190,7 +152,7 @@ describe('RedBlack Tree', function () { var actualResults = 0 this.memory.requestTransaction(function * () { yield* tree.iterate(this, null, upperBound, function * (val) { - expect(val).not.toBeUndefined() + expect(val).toBeDefined() actualResults++ }) expect(expectedResults).toEqual(actualResults) @@ -216,7 +178,7 @@ describe('RedBlack Tree', function () { var actualResults = 0 this.memory.requestTransaction(function * () { yield* tree.iterate(this, lowerBound, upperBound, function * (val) { - expect(val).not.toBeUndefined() + expect(val).toBeDefined() actualResults++ }) expect(expectedResults).toEqual(actualResults) diff --git a/src/Helper.spec.js b/src/Helper.spec.js index 84abc78af27075c78be6f3c149b075b33284fa7a..60643ecb0ca2860ebd1a6ac437b8108e2686e8b4 100644 --- a/src/Helper.spec.js +++ b/src/Helper.spec.js @@ -176,7 +176,7 @@ g.compareAllUsers = async(function * compareAllUsers (users) { var o = yield* this.getOperation([d.id[0], d.id[1] + i]) // gc'd or deleted if (d.gc) { - expect(o).toBeUndefined() + expect(o).toBeNull() } else { expect(o.deleted).toBeTruthy() } diff --git a/src/OperationStores/Memory.spec.js b/src/OperationStores/Memory.spec.js deleted file mode 100644 index 0e8accd3b9a79ca8dd00cec5182a261daf297ee4..0000000000000000000000000000000000000000 --- a/src/OperationStores/Memory.spec.js +++ /dev/null @@ -1,148 +0,0 @@ -/* global Y, async */ -/* eslint-env browser,jasmine,console */ - -describe('Memory', function () { - describe('DeleteStore', function () { - var store - beforeEach(function () { - store = new Y.Memory(null, { - name: 'Memory', - gcTimeout: -1 - }) - }) - it('Deleted operation is deleted', async(function * (done) { - store.requestTransaction(function * () { - yield* this.markDeleted(['u1', 10]) - expect(yield* this.isDeleted(['u1', 10])).toBeTruthy() - expect(yield* this.getDeleteSet()).toEqual({'u1': [[10, 1, false]]}) - done() - }) - })) - it('Deleted operation extends other deleted operation', async(function * (done) { - store.requestTransaction(function * () { - yield* this.markDeleted(['u1', 10]) - yield* this.markDeleted(['u1', 11]) - expect(yield* this.isDeleted(['u1', 10])).toBeTruthy() - expect(yield* this.isDeleted(['u1', 11])).toBeTruthy() - expect(yield* this.getDeleteSet()).toEqual({'u1': [[10, 2, false]]}) - done() - }) - })) - it('Deleted operation extends other deleted operation', async(function * (done) { - store.requestTransaction(function * () { - yield* this.markDeleted(['0', 3]) - yield* this.markDeleted(['0', 4]) - yield* this.markDeleted(['0', 2]) - expect(yield* this.getDeleteSet()).toEqual({'0': [[2, 3, false]]}) - done() - }) - })) - it('Debug #1', async(function * (done) { - store.requestTransaction(function * () { - yield* this.markDeleted(['166', 0]) - yield* this.markDeleted(['166', 2]) - yield* this.markDeleted(['166', 0]) - yield* this.markDeleted(['166', 2]) - yield* this.markGarbageCollected(['166', 2]) - yield* this.markDeleted(['166', 1]) - yield* this.markDeleted(['166', 3]) - yield* this.markGarbageCollected(['166', 3]) - yield* this.markDeleted(['166', 0]) - expect(yield* this.getDeleteSet()).toEqual({'166': [[0, 2, false], [2, 2, true]]}) - done() - }) - })) - it('Debug #2', async(function * (done) { - store.requestTransaction(function * () { - yield* this.markDeleted(['293', 0]) - yield* this.markDeleted(['291', 2]) - yield* this.markDeleted(['291', 2]) - yield* this.markGarbageCollected(['293', 0]) - yield* this.markDeleted(['293', 1]) - yield* this.markGarbageCollected(['291', 2]) - expect(yield* this.getDeleteSet()).toEqual({'291': [[2, 1, true]], '293': [[0, 1, true], [1, 1, false]]}) - done() - }) - })) - it('Debug #3', async(function * (done) { - store.requestTransaction(function * () { - yield* this.markDeleted(['581', 0]) - yield* this.markDeleted(['581', 1]) - yield* this.markDeleted(['580', 0]) - yield* this.markDeleted(['580', 0]) - yield* this.markGarbageCollected(['581', 0]) - yield* this.markDeleted(['581', 2]) - yield* this.markDeleted(['580', 1]) - yield* this.markDeleted(['580', 2]) - yield* this.markDeleted(['580', 1]) - yield* this.markDeleted(['580', 2]) - yield* this.markGarbageCollected(['581', 2]) - yield* this.markGarbageCollected(['581', 1]) - yield* this.markGarbageCollected(['580', 1]) - expect(yield* this.getDeleteSet()).toEqual({'580': [[0, 1, false], [1, 1, true], [2, 1, false]], '581': [[0, 3, true]]}) - done() - }) - })) - it('Debug #4', async(function * (done) { - store.requestTransaction(function * () { - yield* this.markDeleted(['544', 0]) - yield* this.markDeleted(['543', 2]) - yield* this.markDeleted(['544', 0]) - yield* this.markDeleted(['543', 2]) - yield* this.markGarbageCollected(['544', 0]) - yield* this.markDeleted(['545', 1]) - yield* this.markDeleted(['543', 4]) - yield* this.markDeleted(['543', 3]) - yield* this.markDeleted(['544', 1]) - yield* this.markDeleted(['544', 2]) - yield* this.markDeleted(['544', 1]) - yield* this.markDeleted(['544', 2]) - yield* this.markGarbageCollected(['543', 2]) - yield* this.markGarbageCollected(['543', 4]) - yield* this.markGarbageCollected(['544', 2]) - yield* this.markGarbageCollected(['543', 3]) - expect(yield* this.getDeleteSet()).toEqual({'543': [[2, 3, true]], '544': [[0, 1, true], [1, 1, false], [2, 1, true]], '545': [[1, 1, false]]}) - done() - }) - })) - it('Debug #5', async(function * (done) { - store.requestTransaction(function * () { - yield* this.applyDeleteSet({'16': [[1, 2, false]], '17': [[0, 1, true], [1, 3, false]]}) - expect(yield* this.getDeleteSet()).toEqual({'16': [[1, 2, false]], '17': [[0, 1, true], [1, 3, false]]}) - yield* this.applyDeleteSet({'16': [[1, 2, false]], '17': [[0, 4, true]]}) - expect(yield* this.getDeleteSet()).toEqual({'16': [[1, 2, false]], '17': [[0, 4, true]]}) - done() - }) - })) - it('Debug #6', async(function * (done) { - store.requestTransaction(function * () { - yield* this.applyDeleteSet({'40': [[0, 3, false]]}) - expect(yield* this.getDeleteSet()).toEqual({'40': [[0, 3, false]]}) - yield* this.applyDeleteSet({'39': [[2, 2, false]], '40': [[0, 1, true], [1, 2, false]], '41': [[2, 1, false]]}) - expect(yield* this.getDeleteSet()).toEqual({'39': [[2, 2, false]], '40': [[0, 1, true], [1, 2, false]], '41': [[2, 1, false]]}) - done() - }) - })) - it('Debug #7', async(function * (done) { - store.requestTransaction(function * () { - yield* this.markDeleted(['9', 2]) - yield* this.markDeleted(['11', 2]) - yield* this.markDeleted(['11', 4]) - yield* this.markDeleted(['11', 1]) - yield* this.markDeleted(['9', 4]) - yield* this.markDeleted(['10', 0]) - yield* this.markGarbageCollected(['11', 2]) - yield* this.markDeleted(['11', 2]) - yield* this.markGarbageCollected(['11', 3]) - yield* this.markDeleted(['11', 3]) - yield* this.markDeleted(['11', 3]) - yield* this.markDeleted(['9', 4]) - yield* this.markDeleted(['10', 0]) - yield* this.markGarbageCollected(['11', 1]) - yield* this.markDeleted(['11', 1]) - expect(yield* this.getDeleteSet()).toEqual({'9': [[2, 1, false], [4, 1, false]], '10': [[0, 1, false]], '11': [[1, 3, true], [4, 1, false]]}) - done() - }) - })) - }) -}) diff --git a/src/OperationStore.js b/src/Transaction.js similarity index 69% rename from src/OperationStore.js rename to src/Transaction.js index 6842c2d836e4a007ff4d4a37142d8605f10c5788..ad3ce6c3461826547e8661f53dfb187f3dfd0ab0 100644 --- a/src/OperationStore.js +++ b/src/Transaction.js @@ -206,14 +206,14 @@ class AbstractTransaction { // un-extend left var newlen = n.val.len - (id[1] - n.val.id[1]) n.val.len -= newlen - n = yield this.ds.add({id: id, len: newlen, gc: false}) + n = yield this.ds.set({id: id, len: newlen, gc: false}) } // get prev&next before adding a new operation var prev = n.prev() var next = n.next() if (id[1] < n.val.id[1] + n.val.len - 1) { // un-extend right - yield this.ds.add({id: [id[0], id[1] + 1], len: n.val.len - 1, gc: false}) + yield this.ds.set({id: [id[0], id[1] + 1], len: n.val.len - 1, gc: false}) n.val.len = 1 } // set gc'd @@ -256,11 +256,11 @@ class AbstractTransaction { n.val.len++ } else { // cannot extend left - n = yield this.ds.add({id: id, len: 1, gc: false}) + n = yield this.ds.set({id: id, len: 1, gc: false}) } } else { // cannot extend left - n = yield this.ds.add({id: id, len: 1, gc: false}) + n = yield this.ds.set({id: id, len: 1, gc: false}) } // can extend right? var next = n.next() @@ -488,7 +488,7 @@ class AbstractTransaction { return op } * addOperation (op) { - var n = yield this.os.add(op) + var n = yield this.os.set(op) return function () { if (n != null) { n = n.next() @@ -505,10 +505,14 @@ class AbstractTransaction { yield this.os.delete(id) } * setState (state) { - this.ss[state.user] = state.clock + this.ss.set({ + id: [state.user], + clock: state.clock + }) } * getState (user) { - var clock = this.ss[user] + var n + var clock = (n = this.ss.find([user])) == null ? null : n.clock if (clock == null) { clock = 0 } @@ -519,17 +523,20 @@ class AbstractTransaction { } * getStateVector () { var stateVector = [] - for (var user in this.ss) { - var clock = this.ss[user] + yield* this.ss.iterate(this, null, null, function * (n) { stateVector.push({ - user: user, - clock: clock + user: n.id[0], + clock: n.clock }) - } + }) return stateVector } * getStateSet () { - return Y.utils.copyObject(this.ss) + var ss = {} + yield* this.ss.iterate(this, null, null, function * (n) { + ss[n.id[0]] = n.clock + }) + return ss } * getOperations (startSS) { // TODO: use bounds here! @@ -619,284 +626,3 @@ class AbstractTransaction { } } Y.AbstractTransaction = AbstractTransaction - -/* - Partial definition of an OperationStore. - TODO: name it Database, operation store only holds operations. - - A database definition must alse define the following methods: - * logTable() (optional) - - show relevant information information in a table - * requestTransaction(makeGen) - - request a transaction - * destroy() - - destroy the database -*/ -class AbstractOperationStore { - constructor (y, opts) { - this.y = y - // E.g. this.listenersById[id] : Array<Listener> - this.listenersById = {} - // Execute the next time a transaction is requested - this.listenersByIdExecuteNow = [] - // A transaction is requested - this.listenersByIdRequestPending = false - /* To make things more clear, the following naming conventions: - * ls : we put this.listenersById on ls - * l : Array<Listener> - * id : Id (can't use as property name) - * sid : String (converted from id via JSON.stringify - so we can use it as a property name) - - Always remember to first overwrite - a property before you iterate over it! - */ - // TODO: Use ES7 Weak Maps. This way types that are no longer user, - // wont be kept in memory. - this.initializedTypes = {} - this.whenUserIdSetListener = null - - this.gc1 = [] // first stage - this.gc2 = [] // second stage -> after that, remove the op - this.gcTimeout = opts.gcTimeout || 5000 - var os = this - function garbageCollect () { - return new Promise((resolve) => { - os.requestTransaction(function * () { - if (os.y.connector.isSynced) { - for (var i in os.gc2) { - var oid = os.gc2[i] - yield* this.garbageCollectOperation(oid) - } - os.gc2 = os.gc1 - os.gc1 = [] - } - if (os.gcTimeout > 0) { - os.gcInterval = setTimeout(garbageCollect, os.gcTimeout) - } - resolve() - }) - }) - } - this.garbageCollect = garbageCollect - if (this.gcTimeout > 0) { - garbageCollect() - } - } - stopGarbageCollector () { - var self = this - return new Promise(function (resolve) { - self.requestTransaction(function * () { - var ungc = self.gc1.concat(self.gc2) - self.gc1 = [] - self.gc2 = [] - for (var i in ungc) { - var op = yield* this.getOperation(ungc[i]) - delete op.gc - yield* this.setOperation(op) - } - resolve() - }) - }) - } - * garbageCollectAfterSync () { - this.requestTransaction(function * () { - yield* this.os.iterate(this, null, null, function * (op) { - if (op.deleted && op.left != null) { - var left = yield this.os.find(op.left) - this.store.addToGarbageCollector(op, left) - } - }) - }) - } - /* - Try to add to GC. - - TODO: rename this function - - Rulez: - * Only gc if this user is online - * The most left element in a list must not be gc'd. - => There is at least one element in the list - - returns true iff op was added to GC - */ - addToGarbageCollector (op, left) { - if ( - op.gc == null && - op.deleted === true && - this.y.connector.isSynced && - left != null && - left.deleted === true - ) { - op.gc = true - this.gc1.push(op.id) - return true - } else { - return false - } - } - removeFromGarbageCollector (op) { - function filter (o) { - return !Y.utils.compareIds(o, op.id) - } - this.gc1 = this.gc1.filter(filter) - this.gc2 = this.gc2.filter(filter) - delete op.gc - } - destroy () { - clearInterval(this.gcInterval) - this.gcInterval = null - } - setUserId (userId) { - this.userId = userId - this.opClock = 0 - if (this.whenUserIdSetListener != null) { - this.whenUserIdSetListener() - this.whenUserIdSetListener = null - } - } - whenUserIdSet (f) { - if (this.userId != null) { - f() - } else { - this.whenUserIdSetListener = f - } - } - getNextOpId () { - if (this.userId == null) { - throw new Error('OperationStore not yet initialized!') - } - return [this.userId, this.opClock++] - } - /* - Apply a list of operations. - - * get a transaction - * check whether all Struct.*.requiredOps are in the OS - * check if it is an expected op (otherwise wait for it) - * check if was deleted, apply a delete operation after op was applied - */ - apply (ops) { - for (var key in ops) { - var o = ops[key] - var required = Y.Struct[o.struct].requiredOps(o) - this.whenOperationsExist(required, o) - } - } - /* - op is executed as soon as every operation requested is available. - Note that Transaction can (and should) buffer requests. - */ - whenOperationsExist (ids, op) { - if (ids.length > 0) { - let listener = { - op: op, - missing: ids.length - } - - for (let key in ids) { - let id = ids[key] - let sid = JSON.stringify(id) - let l = this.listenersById[sid] - if (l == null) { - l = [] - this.listenersById[sid] = l - } - l.push(listener) - } - } else { - this.listenersByIdExecuteNow.push({ - op: op - }) - } - - if (this.listenersByIdRequestPending) { - return - } - - this.listenersByIdRequestPending = true - var store = this - - this.requestTransaction(function * () { - var exeNow = store.listenersByIdExecuteNow - store.listenersByIdExecuteNow = [] - - var ls = store.listenersById - store.listenersById = {} - - store.listenersByIdRequestPending = false - - for (let key in exeNow) { - let o = exeNow[key].op - yield* store.tryExecute.call(this, o) - } - - for (var sid in ls) { - var l = ls[sid] - var id = JSON.parse(sid) - if ((yield* this.getOperation(id)) == null) { - store.listenersById[sid] = l - } else { - for (let key in l) { - let listener = l[key] - let o = listener.op - if (--listener.missing === 0) { - yield* store.tryExecute.call(this, o) - } - } - } - } - }) - } - /* - Actually execute an operation, when all expected operations are available. - */ - * tryExecute (op) { - if (op.struct === 'Delete') { - yield* Y.Struct.Delete.execute.call(this, op) - } else if ((yield* this.getOperation(op.id)) == null && !(yield* this.isGarbageCollected(op.id))) { - yield* Y.Struct[op.struct].execute.call(this, op) - var next = yield* this.addOperation(op) - yield* this.store.operationAdded(this, op, next) - - // Delete if DS says this is actually deleted - if (yield* this.isDeleted(op.id)) { - yield* Y.Struct['Delete'].execute.call(this, {struct: 'Delete', target: op.id}) - } - } - } - // called by a transaction when an operation is added - * operationAdded (transaction, op, next) { - // increase SS - var o = op - var state = yield* transaction.getState(op.id[0]) - while (o != null && o.id[1] === state.clock && op.id[0] === o.id[0]) { - // either its a new operation (1. case), or it is an operation that was deleted, but is not yet in the OS - state.clock++ - yield* transaction.checkDeleteStoreForState(state) - o = next() - } - yield* transaction.setState(state) - - // notify whenOperation listeners (by id) - var sid = JSON.stringify(op.id) - var l = this.listenersById[sid] - delete this.listenersById[sid] - - if (l != null) { - for (var key in l) { - var listener = l[key] - if (--listener.missing === 0) { - this.whenOperationsExist([], listener.op) - } - } - } - // notify parent, if it has been initialized as a custom type - var t = this.initializedTypes[JSON.stringify(op.parent)] - if (t != null && !op.deleted) { - yield* t._changed(transaction, Y.utils.copyObject(op)) - } - } -} -Y.AbstractOperationStore = AbstractOperationStore