From 3b64c4e4551e70766927fb35b8093f0ff42b3370 Mon Sep 17 00:00:00 2001 From: Ratik Jindal Date: Thu, 1 Sep 2022 18:46:46 +0530 Subject: [PATCH 1/5] feat: apply updates in index.ts --- src/index.ts | 77 ++++++++++++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/src/index.ts b/src/index.ts index 44de653..dcf1ae3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,24 +1,24 @@ import * as Y from 'yjs' import type { Doc as YDoc } from 'yjs' -import type Libp2p from 'libp2p' -import { Uint8ArrayEquals } from './util' -import PeerId from 'peer-id' -import type { BufferList } from 'libp2p-interfaces/src/pubsub' -import { Connection } from 'libp2p-interfaces/src/topology' +import { Libp2p as ILibp2p } from "libp2p"; +import { Uint8ArrayEquals } from './util.js' +import { Connection } from '@libp2p/interface-connection'; // @ts-ignore import * as awarenessProtocol from 'y-protocols/dist/awareness.cjs' import type { Awareness } from 'y-protocols/awareness' +import { peerIdFromString } from "@libp2p/peer-id"; + // the Muxedstream type is wrong for the protocol streams type ProtocolStream = { sink: (data: Iterable | AsyncIterable) => Promise - source: AsyncIterable + source: AsyncIterable close: () => void } function changesTopic(topic: string): string { - return `/marcopolo/gossipPad/${topic}/changes/0.0.01` + return `/marcopolo/gossipPad/${topic}/changes/0.0.01`; } function stateVectorTopic(topic: string): string { @@ -33,9 +33,9 @@ function awarenessProtocolTopic(topic: string): string { return `/marcopolo/gossipPad/${topic}/awareness/0.0.1` } -class Provider { +export class Provider { ydoc: YDoc; - node: Libp2p; + node: ILibp2p; peerID: string; topic: string stateVectors: { [key: string]: Uint8Array } = {}; @@ -46,11 +46,11 @@ class Provider { aggressivelyKeepPeersUpdated: boolean = true; - constructor(ydoc: YDoc, node: Libp2p, topic: string) { + constructor(ydoc: YDoc, node: ILibp2p, topic: string) { this.ydoc = ydoc; this.node = node; this.topic = topic; - this.peerID = this.node.peerId.toB58String() + this.peerID = this.node.peerId.toString() this.stateVectors[this.peerID] = Y.encodeStateVector(this.ydoc) this.awareness = new awarenessProtocol.Awareness(ydoc) @@ -61,14 +61,23 @@ class Provider { ydoc.on('update', this.onUpdate.bind(this)); node.pubsub.subscribe(changesTopic(topic)) - node.pubsub.on(changesTopic(topic), this.onPubSubChanges.bind(this)); - node.pubsub.subscribe(stateVectorTopic(topic)) - node.pubsub.on(stateVectorTopic(topic), this.onPubSubStateVector.bind(this)); - node.pubsub.subscribe(awarenessProtocolTopic(topic)) - node.pubsub.on(awarenessProtocolTopic(topic), this.onPubSubAwareness.bind(this)); + this.node.pubsub.addEventListener('message', async (event) => { + if (event.detail.topic === changesTopic(topic)) { + this.onPubSubChanges.bind(this) + } + + if (event.detail.topic === stateVectorTopic(topic)) { + this.onPubSubStateVector.bind(this) + } + + if (event.detail.topic === awarenessProtocolTopic(topic)) { + this.onPubSubAwareness.bind(this) + } + }); + // @ts-ignore node.handle(syncProtocol(topic), this.onSyncMsg.bind(this)); @@ -77,13 +86,10 @@ class Provider { destroy() { this.node.pubsub.unsubscribe(changesTopic(this.topic)) - this.node.pubsub.removeAllListeners(changesTopic(this.topic)) - this.node.pubsub.unsubscribe(stateVectorTopic(this.topic)) - this.node.pubsub.removeAllListeners(stateVectorTopic(this.topic)) - this.node.pubsub.unsubscribe(awarenessProtocolTopic(this.topic)) - this.node.pubsub.removeAllListeners(awarenessProtocolTopic(this.topic)) + + this.node.pubsub.removeEventListener('message'); // @ts-ignore node.unhandle(syncProtocol(topic)) @@ -92,7 +98,7 @@ class Provider { } // Not required, but nice if we can get synced against a peer sooner rather than latter - private async tryInitialSync(updateData: Uint8Array, origin: this | any) { + private async tryInitialSync(updateData: Uint8Array, origin: this | any): Promise { const tries = 10; const maxWaitTime = 1000; let waitTime = 100; @@ -100,7 +106,9 @@ class Provider { if (this.initialSync) { return } - const peers = [...this.node.pubsub.topics.get(stateVectorTopic(this.topic)) || []] + + // needed to type hack + const peers = [...(this.node as any).pubsub.topics.get(stateVectorTopic(this.topic)) || []] if (peers.length !== 0) { const peer = peers[i % peers.length] @@ -199,7 +207,7 @@ class Provider { } private async onSyncMsg({ stream, connection, ...rest }: { stream: ProtocolStream, connection: Connection }) { - await this.runSyncProtocol(stream, connection.remotePeer.toB58String(), false) + await this.runSyncProtocol(stream, connection.remotePeer.toString(), false) } private queuePeerSync(peerID: string) { @@ -215,22 +223,14 @@ class Provider { private async syncPeer(peerID: string) { const thiz = this; - const multiaddrs = await this.node.peerStore.addressBook.getMultiaddrsForPeer(PeerId.createFromB58String(peerID)); let success = false; - if (!multiaddrs) { + try { + const stream = await this.node.dialProtocol(peerIdFromString(peerID), syncProtocol(this.topic)) as any as { stream: ProtocolStream } + await this.runSyncProtocol(stream as any, peerID, true) + success = true; return - } - for (const ma of multiaddrs) { - const maStr = ma.toString() - try { - const { stream } = await this.node.dialProtocol(maStr, syncProtocol(this.topic)) as any as { stream: ProtocolStream } - await this.runSyncProtocol(stream, peerID, true) - success = true; - return - } catch (e) { - console.warn(`Failed to sync with ${maStr}`, e) - continue; - } + } catch (e) { + console.warn(`Failed to sync with ${peerIdFromString(peerID)}`, e) } throw new Error("Failed to sync with peer") @@ -239,4 +239,3 @@ class Provider { // TODO awareness -export default Provider \ No newline at end of file From 923e51891b685fb7fce3065677370fe025458915 Mon Sep 17 00:00:00 2001 From: Ratik Jindal Date: Thu, 1 Sep 2022 18:47:45 +0530 Subject: [PATCH 2/5] chore: update tsconfig --- tsconfig.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tsconfig.json b/tsconfig.json index c915c5d..5077a52 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -22,9 +22,9 @@ // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ /* Modules */ - "module": "commonjs", /* Specify what module code is generated. */ + "module": "Node16", /* Specify what module code is generated. */ // "rootDir": "./", /* Specify the root folder within your source files. */ - // "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ + "moduleResolution": "Node16", /* Specify how TypeScript looks up a file from a given module specifier. */ // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ From f2d4e15b20250511ae028bc61380a0320618bddc Mon Sep 17 00:00:00 2001 From: Ratik Jindal Date: Thu, 1 Sep 2022 18:48:02 +0530 Subject: [PATCH 3/5] chore: rename babel.config.js to babel.config.cjs --- babel.config.js => babel.config.cjs | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename babel.config.js => babel.config.cjs (100%) diff --git a/babel.config.js b/babel.config.cjs similarity index 100% rename from babel.config.js rename to babel.config.cjs From 38ee88af064feee66020403763a96c143802750a Mon Sep 17 00:00:00 2001 From: Ratik Jindal Date: Thu, 1 Sep 2022 18:49:46 +0530 Subject: [PATCH 4/5] chore: update dependencies in package.json --- package.json | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 2cb2a7b..c415991 100644 --- a/package.json +++ b/package.json @@ -2,20 +2,21 @@ "name": "y-libp2p", "version": "0.0.2", "description": "js-libp2p provider for yjs", - "main": "dist/index.js", + "exports": "./dist/index.js", + "type": "module", "author": "marcopolo", "license": "MIT", "devDependencies": { "@babel/core": "^7.16.0", "@babel/preset-env": "^7.16.4", "@babel/preset-typescript": "^7.16.0", - "@chainsafe/libp2p-noise": "^4.1.1", + "@chainsafe/libp2p-noise": "^8.0.1", + "@libp2p/mplex": "^5.0.0", + "@libp2p/websockets": "^3.0.2", "@types/jest": "^27.0.3", "babel-jest": "^27.4.2", "jest": "^27.4.3", - "libp2p-gossipsub": "^0.11.4", - "libp2p-mplex": "^0.10.4", - "libp2p-websockets": "^0.16.2", + "libp2p-gossipsub": "^0.13.0", "multiaddr": "^10.0.1", "typescript": "^4.5.2" }, @@ -23,13 +24,13 @@ "node": ">=16.0.0" }, "scripts": { - "test": "jest", + "test": "jest dist/", "build": "tsc", "build-watch": "tsc -w" }, "dependencies": { - "y-protocols": "^1.0.x", - "peer-id": ">=0.16.0" + "@libp2p/peer-id": "^1.1.15", + "y-protocols": "^1.0.x" }, "peerDependencies": { "yjs": "13.5.x", From 020b006393f03d0fcd4ef0763485b77f3385591b Mon Sep 17 00:00:00 2001 From: Ratik Jindal Date: Thu, 1 Sep 2022 18:50:08 +0530 Subject: [PATCH 5/5] test: update tests --- src/index.test.ts | 24 ++++++++++++------------ src/test-utils/create-peer.js | 22 +++++++++++----------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/index.test.ts b/src/index.test.ts index 896ce5d..1ccf6b3 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -1,10 +1,10 @@ -import Provider from "./index" +import { Provider } from "./index.js" import { Doc, Doc as YDoc, } from 'yjs' -import { Uint8ArrayEquals } from "./util" +import { Uint8ArrayEquals } from "./util.js" import * as Y from 'yjs' // @ts-ignore import * as awarenessProtocol from 'y-protocols/dist/awareness.cjs' -import Libp2p from 'libp2p' +import {Libp2p as ILibp2p} from 'libp2p' // @ts-ignore import { createPeer } from './test-utils/create-peer.js' @@ -25,8 +25,8 @@ it.only('Provider syncs doc across 2 peers', async () => { const ydoc1 = new YDoc() const ydoc2 = new YDoc() - const node1: Libp2p = await createPeer() - const node2: Libp2p = await createPeer() + const node1: ILibp2p = await createPeer() + const node2: ILibp2p = await createPeer() const provider1 = new Provider(ydoc1, node1, topic) const provider2 = new Provider(ydoc2, node2, topic) @@ -60,8 +60,8 @@ it('Provider syncs doc across 2 unsynced peers', async () => { const ydoc2 = new YDoc() ydoc2.getText("testDoc").insert(0, "Good bye") - const node1: Libp2p = await createPeer() - const node2: Libp2p = await createPeer() + const node1: ILibp2p = await createPeer() + const node2: ILibp2p = await createPeer() const provider1 = new Provider(ydoc1, node1, topic) const provider2 = new Provider(ydoc2, node2, topic) @@ -96,11 +96,11 @@ function printStates(docs: { [key: string]: YDoc }) { console.log("--- Doc States ---" + str) } -async function connectNodes(nodes: Libp2p[]) { +async function connectNodes(nodes: ILibp2p[]) { const firstNode = nodes[0] for (let i = 1; i < nodes.length; i++) { const node = nodes[i] - await firstNode.dial(node.multiaddrs[0] + '/p2p/' + node.peerId.toB58String()) + await firstNode.dial(node.peerId) await firstNode.ping(node.peerId) await node.ping(firstNode.peerId) } @@ -110,7 +110,7 @@ async function connectNodes(nodes: Libp2p[]) { if (i === j) continue const node = nodes[i] const otherNode = nodes[j] - node.peerStore.addressBook.set(otherNode.peerId, otherNode.multiaddrs) + node.peerStore.addressBook.set(otherNode.peerId, otherNode.getMultiaddrs()) } } } @@ -120,8 +120,8 @@ it('Provider syncs awareness across 2 peers', async () => { const ydoc1 = new YDoc() const ydoc2 = new YDoc() - const node1: Libp2p = await createPeer() - const node2: Libp2p = await createPeer() + const node1: ILibp2p = await createPeer() + const node2: ILibp2p = await createPeer() const provider1 = new Provider(ydoc1, node1, topic) const provider2 = new Provider(ydoc2, node2, topic) diff --git a/src/test-utils/create-peer.js b/src/test-utils/create-peer.js index 37a9b92..31e7683 100644 --- a/src/test-utils/create-peer.js +++ b/src/test-utils/create-peer.js @@ -1,4 +1,4 @@ -'use strict' + /** * These utilities rely on the fixtures defined in test/fixtures @@ -10,13 +10,13 @@ const Libp2p = require('libp2p') const { Multiaddr } = require('multiaddr') -const PeerId = require('peer-id') +const PeerId = require('@libp2p/peer-id') const Gossipsub = require('libp2p-gossipsub') -const WS = require('libp2p-websockets') -const filters = require('libp2p-websockets/src/filters') -const MPLEX = require('libp2p-mplex') -const { NOISE } = require('@chainsafe/libp2p-noise') +const WS = require('@libp2p/websockets') +const filters = require('@libp2p/websockets/src/filters') +const Mplex = require('@libp2p/mplex') +const { Noise } = require('@chainsafe/libp2p-noise') const RelayPeer = { id: 'QmckxVrJw1Yo8LqvmDJNUmdAsKtSbiKWmrXJFyKmUraBoN', @@ -50,9 +50,9 @@ const transportKey = WS.prototype[Symbol.toStringTag] const defaultConfig = { modules: { - transport: [WS], - streamMuxer: [MPLEX], - connEncryption: [NOISE], + transports: [new WS()], + streamMuxers: [new Mplex()], + connEncryption: [new Noise()], pubsub: Gossipsub }, config: { @@ -97,7 +97,7 @@ let currentPeerIdx = 0 */ async function createPeer({ peerId, peerIdx = currentPeerIdx, started = true, config = {} } = {}) { if (!peerId) { - peerId = await PeerId.createFromJSON(Peers[currentPeerIdx++ % Peers.length]) + peerId = await PeerId.createPeerId(Peers[currentPeerIdx++ % Peers.length]) } const libp2p = await Libp2p.create({ peerId: peerId, @@ -136,7 +136,7 @@ function addPeersToAddressBook(peers) { */ async function createPeers({ number = 1, started = true, seedAddressBook = true, config = {} } = {}) { const peerIds = await Promise.all( - Array.from({ length: number }, (_, i) => Peers[i] ? PeerId.createFromJSON(Peers[i]) : PeerId.create()) + Array.from({ length: number }, (_, i) => Peers[i] ? PeerId.createPeerId(Peers[i]) : PeerId.createPeerId({ type: "Ed25519"})) ) const peers = await Promise.all( Array.from({ length: number }, (_, i) => createPeer({ peerId: peerIds[i], started: false, config: config }))