From 1b26f4c1b37cfe29f0dcc6d42f9ccbbcd1e7f110 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 30 Mar 2021 12:33:44 +0100 Subject: [PATCH 01/25] feat: store metadata in IPLD CBOR format --- client/package.json | 11 ++++++----- client/src/lib.js | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/client/package.json b/client/package.json index c350806dc8..42ff362254 100644 --- a/client/package.json +++ b/client/package.json @@ -33,6 +33,7 @@ "prepare": "npm run build" }, "dependencies": { + "@ipld/dag-cbor": "^4.0.0", "@web-std/blob": "2.0.1", "@web-std/fetch": "1.0.0", "@web-std/file": "1.0.1", @@ -40,20 +41,20 @@ }, "devDependencies": { "@ssttevee/multipart-parser": "0.1.8", - "uvu": "0.5.1", - "mocha": "8.3.2", "@types/mocha": "8.2.1", "ipfs-unixfs-importer": "6.0.1", "ipld": "0.29.0", "ipld-dag-pb": "0.22.0", "ipld-in-memory": "8.0.0", + "mocha": "8.3.2", "multicodec": "3.0.1", - "multiformats": "4.5.3", + "multiformats": "^4.5.3", "multihashing-async": "2.1.2", "playwright-test": "2.1.0", - "typedoc": "0.20.32", "rollup": "2.22.1", - "rollup-plugin-multi-input": "1.1.1" + "rollup-plugin-multi-input": "1.1.1", + "typedoc": "0.20.32", + "uvu": "0.5.1" }, "homepage": "https://github.com/ipfs-shipyard/nft.storage/tree/main/client", "bugs": "https://github.com/ipfs-shipyard/nft.storage/issues" diff --git a/client/src/lib.js b/client/src/lib.js index 031d2800b5..e802db4997 100644 --- a/client/src/lib.js +++ b/client/src/lib.js @@ -14,6 +14,8 @@ * @module */ +import cbor from '@ipld/dag-cbor' +import { CID } from 'multiformats' import * as API from './lib/interface.js' import { fetch, File, Blob, FormData } from './platform.js' @@ -112,6 +114,39 @@ class NFTStorage { } } + /** + * @param {API.Service} service + * @param {any} metadata + * @returns {Promise} + */ + static async storeMetadata(service, metadata) { + /** + * @param {any} obj + * @returns {Promise} + */ + async function transform (obj) { + if (Array.isArray(obj)) { + return Promise.all(obj.map(transform)) + } + if (obj instanceof File) { + return NFTStorage.storeDirectory(service, [obj]) + } + if (obj instanceof Blob) { + return NFTStorage.storeBlob(service, obj) + } + if (typeof obj === 'object') { + const ents = await Promise.all(Object.entries(obj).map(async ([k, v]) => ([k, await transform(v)]))) + return Object.fromEntries(ents) + } + return obj + } + metadata = await transform(metadata) + const cid = await NFTStorage.storeBlob(service, new Blob([cbor.encode(metadata)])) + console.log(cid) + const mh = CID.parse(cid).multihash + return CID.createV1(cbor.code, mh).toString() + } + /** * @param {API.Service} service * @param {string} cid @@ -175,6 +210,13 @@ class NFTStorage { storeBlob(blob) { return NFTStorage.storeBlob(this, blob) } + /** + * Stores a metadata object as an IPLD CBOR encoded DAG. + * @param {any} metadata + */ + storeMetadata(metadata) { + return NFTStorage.storeMetadata(this, metadata) + } /** * Stores a directory of files and returns a CID for the directory. * From 64c6f6462e9fc4023da3b52a5c39827415e44f31 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Mon, 5 Apr 2021 23:59:50 -0700 Subject: [PATCH 02/25] feat: store api --- client/src/lib.js | 112 +++++++++++++-------- client/src/lib/interface.ts | 166 +++++++++++++++++++++++++++++++ client/src/platform.js | 4 +- client/src/token.js | 188 ++++++++++++++++++++++++++++++++++++ client/test/lib.spec.js | 169 +++++++++++++++++++++++++++++++- client/test/service.js | 82 ++++++++++++++++ 6 files changed, 680 insertions(+), 41 deletions(-) create mode 100644 client/src/token.js diff --git a/client/src/lib.js b/client/src/lib.js index e802db4997..ab98eaee5d 100644 --- a/client/src/lib.js +++ b/client/src/lib.js @@ -13,12 +13,14 @@ * ``` * @module */ - -import cbor from '@ipld/dag-cbor' import { CID } from 'multiformats' import * as API from './lib/interface.js' +import * as Token from './token.js' import { fetch, File, Blob, FormData } from './platform.js' +const GATEWAY = new URL('https://gateway.ipfs.io/') +const ENDPOINT = new URL('https://nft.storage') + /** * @implements API.Service */ @@ -46,7 +48,7 @@ class NFTStorage { * * @param {{token: string, endpoint?:URL}} options */ - constructor({ token, endpoint = new URL('https://nft.storage') }) { + constructor({ token, endpoint = ENDPOINT }) { /** * Authorization token. * @@ -115,38 +117,70 @@ class NFTStorage { } /** + * @template {API.TokenInput} T * @param {API.Service} service - * @param {any} metadata - * @returns {Promise} + * @param {T} data + * @returns {Promise>} */ - static async storeMetadata(service, metadata) { - /** - * @param {any} obj - * @returns {Promise} - */ - async function transform (obj) { - if (Array.isArray(obj)) { - return Promise.all(obj.map(transform)) - } - if (obj instanceof File) { - return NFTStorage.storeDirectory(service, [obj]) - } - if (obj instanceof Blob) { - return NFTStorage.storeBlob(service, obj) - } - if (typeof obj === 'object') { - const ents = await Promise.all(Object.entries(obj).map(async ([k, v]) => ([k, await transform(v)]))) - return Object.fromEntries(ents) + static async store( + { endpoint, token }, + { name, description, image, properties, decimals, localization } + ) { + const url = new URL(`/api/store`, endpoint) + // Just validate that expected field are present + if (typeof name !== 'string') { + throw new TypeError( + 'string property `name` identifying the asset is required' + ) + } + if (typeof description != 'string') { + throw new TypeError( + 'string property `description` describing asset is required' + ) + } + + if (!(image instanceof Blob) || !image.type.startsWith('image/')) { + throw new TypeError( + 'proprety `image` must be a Blob or File object with `image/*` mime type' + ) + } + if (typeof decimals !== 'undefined' && typeof decimals != 'number') { + throw new TypeError('proprety `decimals` must be an integer value') + } + + const body = new FormData() + const data = Token.encode( + { name, description, image, properties, decimals, localization }, + body + ) + + body.set('meta', JSON.stringify(data)) + + const response = await fetch(url.toString(), { + method: 'POST', + headers: NFTStorage.auth(token), + body, + }) + + /** @type {API.StoreResponse} */ + const result = await response.json() + + if (result.ok === true) { + const { value } = result + const data = Token.decode(value.data) + + return { + ipld: CID.parse(value.ipld), + metadata: new URL(value.metadata.href), + data, + embed: Token.embed(data, { + gateway: GATEWAY, + }), } - return obj + } else { + throw new Error(result.error.message) } - metadata = await transform(metadata) - const cid = await NFTStorage.storeBlob(service, new Blob([cbor.encode(metadata)])) - console.log(cid) - const mh = CID.parse(cid).multihash - return CID.createV1(cbor.code, mh).toString() } - /** * @param {API.Service} service * @param {string} cid @@ -210,13 +244,6 @@ class NFTStorage { storeBlob(blob) { return NFTStorage.storeBlob(this, blob) } - /** - * Stores a metadata object as an IPLD CBOR encoded DAG. - * @param {any} metadata - */ - storeMetadata(metadata) { - return NFTStorage.storeMetadata(this, metadata) - } /** * Stores a directory of files and returns a CID for the directory. * @@ -265,9 +292,18 @@ class NFTStorage { delete(cid) { return NFTStorage.delete(this, cid) } + + /** + * @template {API.TokenInput} T + * @param {T} token + * @returns {Promise>} + */ + store(token) { + return NFTStorage.store(this, token) + } } -export { NFTStorage, File, Blob, FormData } +export { NFTStorage, File, Blob, FormData, CID } /** * Just to verify API compatibility. diff --git a/client/src/lib/interface.ts b/client/src/lib/interface.ts index 24ce7fd26f..e71b158496 100644 --- a/client/src/lib/interface.ts +++ b/client/src/lib/interface.ts @@ -1,3 +1,5 @@ +import type { CID } from 'multiformats' + export interface Service { endpoint: URL token: string @@ -9,6 +11,27 @@ export interface Service { export type CIDString = string & {} export interface API { + /** + * Stores given token and all the resources (in form of File or a Blob) it + * references along with a metadata JSON as specificed in (ERC-1155). The + * `token.image` must be either `File` or a `Blob` instance, which whill be + * stored and corresponding content address URL will be saved in metadata + * JSON file under `image` field. + * + * If `token.properties` contain properties with `File` or `Blob` values those + * also get stored and their URLs will be saved in metadata json in their + * place. + * + * Note: URLs for `File` object will retain the name e.g. in case of + * `new File([bytes], 'cat.png', { type: 'image/png' })` it will look like + * `ipfs://bafy...hash/image/cat.png`. For `Blob` object URL will not have + * name or mime type instead it will look more like `ipfs://bafy...hash/image/blob` + */ + store( + service: Service, + token: T + ): Promise> + /** * Stores a single file and returns a corresponding CID. */ @@ -106,3 +129,146 @@ export interface Pin { } export type PinStatus = 'queued' | 'pinning' | 'pinned' | 'failed' + +/** + * This is an input used to construct the Token metadata as per EIP-1155 + * @see https://eips.ethereum.org/EIPS/eip-1155#metadata + */ +export interface TokenInput { + /** + * Identifies the asset to which this token represents + */ + name: string + /** + * Describes the asset to which this token represents + */ + description: string + /** + * An `File` with mime type image/* representing the asset this + * token represents. Consider making any images at a width between `320` and + * `1080` pixels and aspect ratio between `1.91:1` and `4:5` inclusive. + * + * If `File` object is used, URL in the metadata will include a filename + * e.g. `ipfs://bafy...hash/cat.png`. If `Blob` is used URL in the metadata + * will not include filename or extension e.g. `ipfs://bafy...img/` + */ + image: Blob | File + + /** + * The number of decimal places that the token amount should display - e.g. + * `18`, means to divide the token amount by `1000000000000000000` to get its + * user representation. + */ + decimals?: number + + /** + * Arbitrary properties. Values may be strings, numbers, nested object or + * arrays of values. It is possible to provide a `File` or a `Blob` instance + * as property value, in which case it is stored on IPFS and metadata will + * contain URL to it in form of `ipfs://bafy...hash/name.png` or + * `ipfs://bafy...file/` respectively. + */ + properties?: Object + + localization?: Localization +} + +interface Localization { + /** + * The URI pattern to fetch localized data from. This URI should contain the + * substring `{locale}` which will be replaced with the appropriate locale + * value before sending the request. + */ + uri: string + /** + * The locale of the default data within the base JSON + */ + default: string + /** + * The list of locales for which data is available. These locales should + * conform to those defined in the Unicode Common Locale Data Repository + * (http://cldr.unicode.org/). + */ + locales: string[] +} + +export interface StoreResult { + /** + * CID for the token that encloses all of the files including metadata.json + * for the stored token. + */ + ipld: CID + + /** + * URL like `ipfs://bafy...hash/meta/data.json` for the stored token metadata. + */ + metadata: URL + + /** + * Actual token data in ERC-1155 format. It is matches data passed as `token` + * argument except Files/Blobs are substituted with corresponding `ipfs://` + * URLs. + */ + data: Encoded + + embed: Encoded +} + +export type EncodedError = { + message: string +} +export type EncodedURL = { + '@': 'URL' + href: string +} + +export type Result = { ok: true; value: T } | { ok: false; error: X } + +export type StoreResponse = Result< + EncodedError, + { + ipld: CIDString + metadata: EncodedURL + data: Encoded + } +> + +/** + * Represents `T` encoded with a given `Format`. + * @example + * ```ts + * type Format = [ + * [URL, { type: 'URL', href: string }] + * [CID, { type: 'CID', cid: string }] + * [Blob, { type: 'Blob', href: string }] + * ] + * + * type Response = Encoded + * ``` + */ +export type Encoded[]> = MatchRecord< + T, + Rule +> + +/** + * Format consists of multiple encoding defines what input type `I` maps to what output type `O`. It + * can be represented via function type or a [I, O] tuple. + */ +type Pattern = ((input: I) => O) | [I, O] + +export type MatchRecord> = { + [K in keyof T]: MatchRule extends never // R extends {I: T[K], O: infer O} ? O : MatchRecord //Match + ? MatchRecord + : MatchRule +} + +type MatchRule> = R extends (input: T) => infer O + ? O + : never + +type Rule> = Format extends [infer I, infer O] + ? (input: I) => O + : Format extends (input: infer I) => infer O + ? (input: I) => O + : never diff --git a/client/src/platform.js b/client/src/platform.js index 1ac5be02d8..adb58e5bef 100644 --- a/client/src/platform.js +++ b/client/src/platform.js @@ -1,7 +1,7 @@ import fetch, { Request, Response, Headers } from '@web-std/fetch' import { FormData } from '@web-std/form-data' -import { Blob, ReadableStream } from '@web-std/blob' -import { File } from '@web-std/file' +import { ReadableStream } from '@web-std/blob' +import { File, Blob } from '@web-std/file' export { fetch, diff --git a/client/src/token.js b/client/src/token.js new file mode 100644 index 0000000000..d3ca64e19d --- /dev/null +++ b/client/src/token.js @@ -0,0 +1,188 @@ +import * as API from './lib/interface.js' +import { Blob } from './platform.js' + +/** + * @template T + * @param {API.Encoded} input + * @param {EmbedOption} options + * @returns {API.Encoded} + */ +export const embed = (input, options) => + mapWith(input, isURL, embedURL, options) + +/** + * @template {API.TokenInput} T + * @param {API.Encoded} value + * @returns {API.Encoded} + */ +export const decode = (value) => mapWith(value, isEncodedURL, decodeURL, null) + +/** + * @param {any} value + * @returns {value is URL} + */ +const isURL = (value) => value instanceof URL + +/** + * @template State + * @param {State} state + * @param {API.EncodedURL} url + * @returns {[State, URL]} + */ +const decodeURL = (state, { href }) => [state, new URL(href)] + +/** + * @typedef {{gateway: URL}} EmbedOption + * + * @param {EmbedOption} context + * @param {URL} url + * @returns {[EmbedOption, URL]} + */ +const embedURL = (context, url) => [ + context, + new URL(`/ipfs/${url.href.slice('ipfs://'.length)}`, context.gateway), +] + +// /** +// * @template T +// * @param {T} value +// * @returns {T} +// */ +// export const decode = (value) => +// // @ts-ignore +// Array.isArray(value) +// ? value.map(decode) +// : isEncodedURL(value) +// ? decodeURL(value) +// : isObject(value) +// ? decodeObject(value) +// : value + +/** + * @param {any} value + * @returns {value is object} + */ +const isObject = (value) => typeof value === 'object' && value != null + +/** + * @param {any} value + * @returns {value is API.EncodedURL} + */ +const isEncodedURL = (value) => + value != null && value['@'] === 'URL' && typeof value.href === 'string' + +/** + * @template {API.TokenInput} T + * @param {API.Encoded} input + * @param {FormData} data + * @returns {API.Encoded} + */ +export const encode = (input, data) => mapWith(input, isBlob, encodeBlob, data) + +/** + * @param {FormData} data + * @param {Blob} blob + * @param {PropertyKey[]} path + * @returns {[FormData, void]} + */ +const encodeBlob = (data, blob, path) => { + data.set(path.join('.'), blob) + return [data, undefined] +} + +/** + * @param {any} value + * @returns {value is Blob} + */ +const isBlob = (value) => value instanceof Blob + +/** + * Substitues values in the given `input` that match `p(value) == true` with + * `f(value, context, path)` where `context` is whatever you pass (usually + * a mutable state) and `path` is a array of keys / indexes where the value + * was encountered. + * + * @template T, I, X, O, State + * @param {API.Encoded} input - Arbitrary input. + * @param {(input:any) => input is X} p - Predicate function to determine + * which values to swap. + * @param {(state:State, input:X, path:PropertyKey[]) => [State, O]} f - Function + * that swaps matching values. + * @param {State} state - Some additional context you need in the process. + * likey you'll start with `[]`. + * @returns {API.Encoded} + */ + +export const mapWith = (input, p, f, state) => { + const [, output] = mapValueWith(input, p, f, state, []) + return output +} + +/** + * @template T, I, X, O, State + * @param {API.Encoded} input - Arbitrary input. + * @param {(input:any) => input is X} p - Predicate function to determine + * which values to swap. + * @param {(state:State, input:X, path:PropertyKey[]) => [State, O]} f - Function + * that swaps matching values. + * @param {State} state - Some additional context you need in the process. + * @param {PropertyKey[]} path - Path where the value was encountered. Most + * likey you'll start with `[]`. + * @returns {[State, API.Encoded]} + */ +const mapValueWith = (input, p, f, state, path) => + p(input) + ? f(state, input, path) + : Array.isArray(input) + ? mapArrayWith(input, p, f, state, path) + : isObject(input) + ? mapObjectWith(input, p, f, state, path) + : [state, /** @type {any} */ (input)] + +/** + * Just like `mapWith` except + * + * @template State, T, I, X, O + * @param {API.Encoded} input + * @param {(input:any) => input is X} p + * @param {(state: State, input:X, path:PropertyKey[]) => [State, O]} f + * @param {State} init + * @param {PropertyKey[]} path + * @returns {[State, API.Encoded]} + */ +const mapObjectWith = (input, p, f, init, path) => { + let state = init + const output = /** @type {API.Encoded} */ ({}) + for (const [key, value] of Object.entries(input)) { + const [next, out] = mapValueWith(value, p, f, state, [...path, key]) + // @ts-ignore + output[key] = out + state = next + } + return [state, output] +} + +/** + * Just like `mapWith` except for Arrays. + * + * @template I, X, O, State + * @template {any[]} T + * @param {T} input + * @param {(input:any) => input is X} p + * @param {(state: State, input:X, path:PropertyKey[]) => [State, O]} f + * @param {State} init + * @param {PropertyKey[]} path + * @returns {[State, API.Encoded]} + */ +const mapArrayWith = (input, p, f, init, path) => { + const output = /** @type {unknown[]} */ ([]) + + let state = init + for (const [index, element] of input.entries()) { + const [next, out] = mapValueWith(element, p, f, state, [...path, index]) + output[index] = out + state = next + } + + return [state, /** @type {API.Encoded} */ (output)] +} diff --git a/client/test/lib.spec.js b/client/test/lib.spec.js index 36fd6afe81..015fcd55af 100644 --- a/client/test/lib.spec.js +++ b/client/test/lib.spec.js @@ -1,5 +1,5 @@ import * as assert from 'uvu/assert' -import { NFTStorage, Blob, File } from 'nft.storage' +import { NFTStorage, Blob, File, CID } from 'nft.storage' describe('client', () => { const { AUTH_TOKEN, SERVICE_ENDPOINT } = process.env @@ -89,6 +89,173 @@ describe('client', () => { }) }) + describe('store', async () => { + it('requires name', async () => { + const client = new NFTStorage({ token, endpoint }) + try { + // @ts-expect-error + await client.store({}) + assert.unreachable('sholud have failed') + } catch (error) { + assert.ok(error instanceof TypeError) + assert.match( + error, + /string property `name` identifying the asset is required/ + ) + } + }) + + it('requires description', async () => { + const client = new NFTStorage({ token, endpoint }) + try { + // @ts-expect-error + await client.store({ name: 'name' }) + assert.unreachable('sholud have failed') + } catch (error) { + assert.ok(error instanceof TypeError) + assert.match( + error, + /string property `description` describing asset is required/ + ) + } + }) + + it('requires image', async () => { + const client = new NFTStorage({ token, endpoint }) + try { + // @ts-expect-error + await client.store({ name: 'name', description: 'stuff' }) + assert.unreachable('sholud have failed') + } catch (error) { + assert.ok(error instanceof TypeError) + assert.match(error, /proprety `image` must be a Blob or File/) + } + }) + + it('requires image mime type', async () => { + const client = new NFTStorage({ token, endpoint }) + try { + await client.store({ + name: 'name', + description: 'stuff', + image: new Blob(['bla bla']), + }) + } catch (error) { + assert.ok(error instanceof TypeError) + assert.match(error, /Blob or File object with `image\/\*` mime type/) + } + }) + + it('expects decimal to be an int', async () => { + const client = new NFTStorage({ token, endpoint }) + try { + await client.store({ + name: 'name', + description: 'stuff', + image: new Blob(['pretend image'], { type: 'image/png' }), + // @ts-expect-error + decimals: 'foo', + }) + } catch (error) { + assert.ok(error instanceof TypeError) + assert.match(error, /proprety `decimals` must be an integer value/) + } + }) + + it('errors without token', async () => { + const client = new NFTStorage({ token: 'wrong', endpoint }) + + try { + await client.store({ + name: 'name', + description: 'tada', + image: new Blob([], { type: 'image/png' }), + }) + assert.unreachable('sholud have failed') + } catch (error) { + assert.ok(error instanceof Error) + assert.match(error, /Unauthorized/) + } + }) + + it('uploads image', async () => { + const client = new NFTStorage({ token, endpoint }) + const result = await client.store({ + name: 'name', + description: 'stuff', + image: new Blob(['fake image'], { type: 'image/png' }), + }) + + assert.ok(result.metadata instanceof URL) + assert.ok(result.metadata.protocol, 'ipfs:') + + assert.ok(result.ipld instanceof CID) + + assert.equal(result.data.name, 'name') + assert.equal(result.data.description, 'stuff') + assert.ok(result.data.image instanceof URL) + assert.ok(result.data.image.protocol, 'ipfs:') + + assert.equal(result.embed.name, 'name') + assert.equal(result.embed.description, 'stuff') + assert.ok(result.embed.image instanceof URL) + assert.ok(result.embed.image.protocol, 'https:') + }) + + it('store with properties', async () => { + const client = new NFTStorage({ token, endpoint }) + const result = await client.store({ + name: 'name', + description: 'stuff', + image: new File(['fake image'], 'cat.png', { type: 'image/png' }), + properties: { + extra: 'meta', + src: [ + new File(['hello'], 'hello.txt', { type: 'text/plain' }), + new Blob(['bye']), + ], + }, + }) + + assert.ok(result.metadata instanceof URL) + assert.ok(result.metadata.protocol, 'ipfs:') + + assert.ok(result.ipld instanceof CID) + + assert.equal(result.data.name, 'name') + assert.equal(result.data.description, 'stuff') + assert.ok(result.data.image instanceof URL) + assert.ok(result.data.image.protocol, 'ipfs:') + + assert.equal(result.data.properties.extra, 'meta') + assert.ok(Array.isArray(result.data.properties.src)) + assert.equal(result.data.properties.src.length, 2) + + const [h, b] = /** @type {[URL, URL]} */ (result.data.properties.src) + assert.ok(h instanceof URL) + assert.equal(h.protocol, 'ipfs:') + + assert.ok(b instanceof URL) + assert.equal(b.protocol, 'ipfs:') + + assert.equal(result.embed.name, 'name') + assert.equal(result.embed.description, 'stuff') + assert.ok(result.embed.image instanceof URL) + assert.ok(result.embed.image.protocol, 'https:') + + assert.equal(result.embed.properties.extra, 'meta') + assert.ok(Array.isArray(result.embed.properties.src)) + assert.equal(result.embed.properties.src.length, 2) + + const [h2, b2] = /** @type {[URL, URL]} */ (result.embed.properties.src) + assert.ok(h2 instanceof URL) + assert.equal(h2.protocol, 'https:') + + assert.ok(b2 instanceof URL) + assert.equal(b2.protocol, 'https:') + }) + }) + describe('status', () => { const client = new NFTStorage({ token, endpoint }) diff --git a/client/test/service.js b/client/test/service.js index 15d6786a3f..994bc9d895 100644 --- a/client/test/service.js +++ b/client/test/service.js @@ -1,6 +1,9 @@ import { CID } from 'multiformats' +import { File } from '../src/platform.js' +import { sha256 } from 'multiformats/hashes/sha2' import { importBlob, importDirectory } from './importer.js' import { Response, Request } from './mock-server.js' +import CBOR from '@ipld/dag-cbor' /** * @param {Request} request */ @@ -32,6 +35,79 @@ const importUpload = async (request) => { } } +/** + * @param {File} file + * @returns {Promise} + */ +const importAsset = async (file) => { + const { cid } = await importDirectory([file]) + return CID.parse(cid.toString()) +} + +/** + * @param {Request} request + */ +const importToken = async (request) => { + const contentType = request.headers.get('content-type') || '' + if (contentType.includes('multipart/form-data')) { + const form = await request.formData() + + const data = JSON.parse(/** @type {string} */ (form.get('meta'))) + const dag = JSON.parse(JSON.stringify(data)) + const metadata = JSON.parse(JSON.stringify(data)) + + for (const [path, content] of form.entries()) { + if (path !== 'meta') { + const file = /** @type {File} */ (content) + const cid = await importAsset(file) + const href = `ipfs://${cid}/${file.name}` + setAt(path.split('.'), dag, cid) + setAt(path.split('.'), data, { '@': 'URL', href }) + setAt(path.split('.'), metadata, href) + } + } + + dag.meta = await importAsset( + new File([JSON.stringify(metadata)], 'data.json') + ) + + const bytes = CBOR.encode(dag) + const hash = await sha256.digest(bytes) + const cid = CID.create(1, CBOR.code, hash) + + const result = { + ok: true, + value: { + ipld: cid.toString(), + metadata: { '@': 'URL', href: `ipfs://${dag.meta}/data.json` }, + data, + }, + } + + return result + } else { + throw Error('/api/store expects multipart/form-data') + } +} + +/** + * @template V + * @param {string[]} path + * @param {any} object + * @param {V} value + */ +const setAt = (path, object, value) => { + const n = path.length - 1 + let target = object + for (let [index, key] of path.entries()) { + if (index === n) { + target[key] = value + } else { + target = target[key] + } + } +} + /** * @typedef {{AUTH_TOKEN:string, store: Map}} State * @returns {State} @@ -72,6 +148,12 @@ export const handle = async (request, { store, AUTH_TOKEN }) => { } try { switch (`${request.method} /${api}/${param}`) { + case 'POST /api/store': { + const result = await importToken(request) + return new Response(JSON.stringify(result), { + headers: headers(request), + }) + } case 'POST /api/upload': { const { cid } = await importUpload(request) const key = `${token}:${cid}` From 9aa46249fc849c93f3edc031ac1cb98112217d34 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Tue, 6 Apr 2021 15:57:46 -0700 Subject: [PATCH 03/25] Apply suggestions from code review Co-authored-by: Alan Shaw --- client/package.json | 2 +- client/src/lib.js | 4 ++-- client/src/lib/interface.ts | 4 ++-- client/test/service.js | 9 +++++++++ 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/client/package.json b/client/package.json index ded84683c1..481bbbf211 100644 --- a/client/package.json +++ b/client/package.json @@ -34,7 +34,6 @@ "prepare": "npm run build" }, "dependencies": { - "@ipld/dag-cbor": "^4.0.0", "@web-std/blob": "2.0.1", "@web-std/fetch": "1.0.0", "@web-std/file": "1.0.1", @@ -51,6 +50,7 @@ "mocha": "8.3.2", "multicodec": "3.0.1", "multiformats": "^4.5.3", + "@ipld/dag-cbor": "^4.0.0", "multihashing-async": "2.1.2", "nyc": "15.1.0", "playwright-test": "2.1.0", diff --git a/client/src/lib.js b/client/src/lib.js index ab98eaee5d..b7085f88bb 100644 --- a/client/src/lib.js +++ b/client/src/lib.js @@ -133,7 +133,7 @@ class NFTStorage { 'string property `name` identifying the asset is required' ) } - if (typeof description != 'string') { + if (typeof description !== 'string') { throw new TypeError( 'string property `description` describing asset is required' ) @@ -144,7 +144,7 @@ class NFTStorage { 'proprety `image` must be a Blob or File object with `image/*` mime type' ) } - if (typeof decimals !== 'undefined' && typeof decimals != 'number') { + if (typeof decimals !== 'undefined' && typeof decimals !== 'number') { throw new TypeError('proprety `decimals` must be an integer value') } diff --git a/client/src/lib/interface.ts b/client/src/lib/interface.ts index e71b158496..53f9c5c314 100644 --- a/client/src/lib/interface.ts +++ b/client/src/lib/interface.ts @@ -14,7 +14,7 @@ export interface API { /** * Stores given token and all the resources (in form of File or a Blob) it * references along with a metadata JSON as specificed in (ERC-1155). The - * `token.image` must be either `File` or a `Blob` instance, which whill be + * `token.image` must be either `File` or a `Blob` instance, which will be * stored and corresponding content address URL will be saved in metadata * JSON file under `image` field. * @@ -205,7 +205,7 @@ export interface StoreResult { metadata: URL /** - * Actual token data in ERC-1155 format. It is matches data passed as `token` + * Actual token data in ERC-1155 format. It matches data passed as `token` * argument except Files/Blobs are substituted with corresponding `ipfs://` * URLs. */ diff --git a/client/test/service.js b/client/test/service.js index 994bc9d895..930a576bab 100644 --- a/client/test/service.js +++ b/client/test/service.js @@ -91,6 +91,15 @@ const importToken = async (request) => { } /** + * Sets a given `value` at the given `path` on a passed `object`. + * + * @example + * ```js + * const obj = { a: { b: { c: 1 }}} + * setAt('a.b.c', obj, 5) + * obj.a.b.c //> 5 + * ``` + * * @template V * @param {string[]} path * @param {any} object From 6cb9476d44725fddedcc8e802ae1780d6a2ce454 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Tue, 6 Apr 2021 16:25:05 -0700 Subject: [PATCH 04/25] chore: change Token.encode to return FormData --- client/src/lib.js | 15 ++++++----- client/src/token.js | 63 +++++++++++++++++++++++++++++++-------------- 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/client/src/lib.js b/client/src/lib.js index b7085f88bb..babcb7bbe6 100644 --- a/client/src/lib.js +++ b/client/src/lib.js @@ -148,13 +148,14 @@ class NFTStorage { throw new TypeError('proprety `decimals` must be an integer value') } - const body = new FormData() - const data = Token.encode( - { name, description, image, properties, decimals, localization }, - body - ) - - body.set('meta', JSON.stringify(data)) + const body = Token.encode({ + name, + description, + image, + properties, + decimals, + localization, + }) const response = await fetch(url.toString(), { method: 'POST', diff --git a/client/src/token.js b/client/src/token.js index d3ca64e19d..b61e1f9284 100644 --- a/client/src/token.js +++ b/client/src/token.js @@ -1,5 +1,5 @@ import * as API from './lib/interface.js' -import { Blob } from './platform.js' +import { Blob, FormData } from './platform.js' /** * @template T @@ -43,21 +43,6 @@ const embedURL = (context, url) => [ new URL(`/ipfs/${url.href.slice('ipfs://'.length)}`, context.gateway), ] -// /** -// * @template T -// * @param {T} value -// * @returns {T} -// */ -// export const decode = (value) => -// // @ts-ignore -// Array.isArray(value) -// ? value.map(decode) -// : isEncodedURL(value) -// ? decodeURL(value) -// : isObject(value) -// ? decodeObject(value) -// : value - /** * @param {any} value * @returns {value is object} @@ -72,12 +57,52 @@ const isEncodedURL = (value) => value != null && value['@'] === 'URL' && typeof value.href === 'string' /** + * Takes token input and encodes it into + * [FormData](https://developer.mozilla.org/en-US/docs/Web/API/FormData) + * object where form field values are discovered `Blob` (or `File`) objects in + * the given token and field keys are `.` joined paths where they were discoverd + * in the token. Additionally encoded `FormData` will also have a field + * named `meta` containing JSON serialized token with blobs and file values + * `null` set to null (this allows backend to injest all of the files from + * `multipart/form-data` request and update provided "meta" data with + * corresponding file ipfs:// URLs) + * + * @example + * ```js + * const cat = new File([], 'cat.png') + * const kitty = new File([], 'kitty.png') + * const form = encode({ + * name: 'hello' + * image: cat + * properties: { + * extra: { + * image: kitty + * } + * } + * }) + * [...form.entries()] //> + * // [ + * // ['image', cat], + * // ['properties.extra.image', kitty], + * // ['meta', '{"name":"hello",image:null,"properties":{"extra":{"kitty": null}}}'] + * // ] + * ``` + * * @template {API.TokenInput} T * @param {API.Encoded} input - * @param {FormData} data - * @returns {API.Encoded} + * @returns {FormData} */ -export const encode = (input, data) => mapWith(input, isBlob, encodeBlob, data) +export const encode = (input) => { + const [form, meta] = mapValueWith( + input, + isBlob, + encodeBlob, + new FormData(), + [] + ) + form.set('meta', JSON.stringify(meta)) + return form +} /** * @param {FormData} data From c1aff1f6cfefdbfd81e9b0da63bfd9f5b1707455 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Tue, 6 Apr 2021 16:25:28 -0700 Subject: [PATCH 05/25] chore: remove CID dependency --- client/src/lib.js | 5 ++--- client/src/lib/interface.ts | 2 +- client/test/lib.spec.js | 9 ++++++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/client/src/lib.js b/client/src/lib.js index babcb7bbe6..8de0b19262 100644 --- a/client/src/lib.js +++ b/client/src/lib.js @@ -13,7 +13,6 @@ * ``` * @module */ -import { CID } from 'multiformats' import * as API from './lib/interface.js' import * as Token from './token.js' import { fetch, File, Blob, FormData } from './platform.js' @@ -171,7 +170,7 @@ class NFTStorage { const data = Token.decode(value.data) return { - ipld: CID.parse(value.ipld), + ipld: value.ipld, metadata: new URL(value.metadata.href), data, embed: Token.embed(data, { @@ -304,7 +303,7 @@ class NFTStorage { } } -export { NFTStorage, File, Blob, FormData, CID } +export { NFTStorage, File, Blob, FormData } /** * Just to verify API compatibility. diff --git a/client/src/lib/interface.ts b/client/src/lib/interface.ts index 53f9c5c314..fc4a22778e 100644 --- a/client/src/lib/interface.ts +++ b/client/src/lib/interface.ts @@ -197,7 +197,7 @@ export interface StoreResult { * CID for the token that encloses all of the files including metadata.json * for the stored token. */ - ipld: CID + ipld: CIDString /** * URL like `ipfs://bafy...hash/meta/data.json` for the stored token metadata. diff --git a/client/test/lib.spec.js b/client/test/lib.spec.js index 015fcd55af..7c9da69c81 100644 --- a/client/test/lib.spec.js +++ b/client/test/lib.spec.js @@ -1,5 +1,6 @@ import * as assert from 'uvu/assert' -import { NFTStorage, Blob, File, CID } from 'nft.storage' +import { NFTStorage, Blob, File } from 'nft.storage' +import { CID } from 'multiformats' describe('client', () => { const { AUTH_TOKEN, SERVICE_ENDPOINT } = process.env @@ -189,7 +190,8 @@ describe('client', () => { assert.ok(result.metadata instanceof URL) assert.ok(result.metadata.protocol, 'ipfs:') - assert.ok(result.ipld instanceof CID) + const cid = CID.parse(result.ipld) + assert.equal(cid.version, 1) assert.equal(result.data.name, 'name') assert.equal(result.data.description, 'stuff') @@ -220,7 +222,8 @@ describe('client', () => { assert.ok(result.metadata instanceof URL) assert.ok(result.metadata.protocol, 'ipfs:') - assert.ok(result.ipld instanceof CID) + const cid = CID.parse(result.ipld) + assert.equal(cid.version, 1) assert.equal(result.data.name, 'name') assert.equal(result.data.description, 'stuff') From 89b5f645d4ca06de4451e9bcc6499764641d522f Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Tue, 6 Apr 2021 16:28:04 -0700 Subject: [PATCH 06/25] chore: add comment for `embed` field --- client/src/lib/interface.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/src/lib/interface.ts b/client/src/lib/interface.ts index fc4a22778e..1fd8c1fc91 100644 --- a/client/src/lib/interface.ts +++ b/client/src/lib/interface.ts @@ -211,6 +211,11 @@ export interface StoreResult { */ data: Encoded + /** + * Token data just like in `data` field except urls corresponding to + * Files/Blobs are substituted with IPFS gateway URLs so they can be + * embedded in browsers that do not support `ipfs://` protocol. + */ embed: Encoded } From 90c3ed7c0136b163452d2d0869a44f1cffae0e2e Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Tue, 6 Apr 2021 16:40:58 -0700 Subject: [PATCH 07/25] chore: rename setAt to setIn --- client/src/token.js | 1 - client/test/service.js | 19 ++++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/client/src/token.js b/client/src/token.js index b61e1f9284..de8c6b932e 100644 --- a/client/src/token.js +++ b/client/src/token.js @@ -137,7 +137,6 @@ const isBlob = (value) => value instanceof Blob * likey you'll start with `[]`. * @returns {API.Encoded} */ - export const mapWith = (input, p, f, state) => { const [, output] = mapValueWith(input, p, f, state, []) return output diff --git a/client/test/service.js b/client/test/service.js index 930a576bab..269a8ec0d5 100644 --- a/client/test/service.js +++ b/client/test/service.js @@ -56,14 +56,15 @@ const importToken = async (request) => { const dag = JSON.parse(JSON.stringify(data)) const metadata = JSON.parse(JSON.stringify(data)) - for (const [path, content] of form.entries()) { - if (path !== 'meta') { + for (const [name, content] of form.entries()) { + if (name !== 'meta') { const file = /** @type {File} */ (content) const cid = await importAsset(file) const href = `ipfs://${cid}/${file.name}` - setAt(path.split('.'), dag, cid) - setAt(path.split('.'), data, { '@': 'URL', href }) - setAt(path.split('.'), metadata, href) + const path = name.split('.') + setIn(dag, path, cid) + setIn(data, path, { '@': 'URL', href }) + setIn(metadata, path, href) } } @@ -96,16 +97,16 @@ const importToken = async (request) => { * @example * ```js * const obj = { a: { b: { c: 1 }}} - * setAt('a.b.c', obj, 5) + * setIn(obj, ['a', 'b', 'c'], 5) * obj.a.b.c //> 5 * ``` - * + * * @template V - * @param {string[]} path * @param {any} object + * @param {string[]} path * @param {V} value */ -const setAt = (path, object, value) => { +const setIn = (object, path, value) => { const n = path.length - 1 let target = object for (let [index, key] of path.entries()) { From 5f499dcc60402a2648b6126d25adbc6b2e53be34 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Tue, 6 Apr 2021 17:11:37 -0700 Subject: [PATCH 08/25] chore: change how URLs are encoded/decoded --- client/src/lib.js | 5 +++-- client/src/lib/interface.ts | 14 +++++++++----- client/src/token.js | 22 +++++++++++++--------- client/test/lib.spec.js | 4 ++++ client/test/service.js | 10 +++------- 5 files changed, 32 insertions(+), 23 deletions(-) diff --git a/client/src/lib.js b/client/src/lib.js index 8de0b19262..0c2aedadae 100644 --- a/client/src/lib.js +++ b/client/src/lib.js @@ -155,6 +155,7 @@ class NFTStorage { decimals, localization, }) + const paths = new Set(body.keys()) const response = await fetch(url.toString(), { method: 'POST', @@ -167,11 +168,11 @@ class NFTStorage { if (result.ok === true) { const { value } = result - const data = Token.decode(value.data) + const data = Token.decode(value.data, paths) return { ipld: value.ipld, - metadata: new URL(value.metadata.href), + metadata: new URL(value.metadata), data, embed: Token.embed(data, { gateway: GATEWAY, diff --git a/client/src/lib/interface.ts b/client/src/lib/interface.ts index 1fd8c1fc91..390f017cb1 100644 --- a/client/src/lib/interface.ts +++ b/client/src/lib/interface.ts @@ -1,5 +1,12 @@ import type { CID } from 'multiformats' +export type { CID } + +/** + * Define nominal type of U based on type of T. Similar to Opaque types in Flow + */ +export type Tagged = T & { tag?: Tag } + export interface Service { endpoint: URL token: string @@ -8,7 +15,7 @@ export interface Service { /** * CID in string representation */ -export type CIDString = string & {} +export type CIDString = Tagged export interface API { /** @@ -222,10 +229,7 @@ export interface StoreResult { export type EncodedError = { message: string } -export type EncodedURL = { - '@': 'URL' - href: string -} +export type EncodedURL = Tagged export type Result = { ok: true; value: T } | { ok: false; error: X } diff --git a/client/src/token.js b/client/src/token.js index de8c6b932e..cc9d5a7516 100644 --- a/client/src/token.js +++ b/client/src/token.js @@ -13,9 +13,11 @@ export const embed = (input, options) => /** * @template {API.TokenInput} T * @param {API.Encoded} value + * @param {Set} paths - Paths were to expcet EncodedURLs * @returns {API.Encoded} */ -export const decode = (value) => mapWith(value, isEncodedURL, decodeURL, null) +export const decode = (value, paths) => + mapWith(value, isEncodedURL, decodeURL, paths) /** * @param {any} value @@ -29,7 +31,7 @@ const isURL = (value) => value instanceof URL * @param {API.EncodedURL} url * @returns {[State, URL]} */ -const decodeURL = (state, { href }) => [state, new URL(href)] +const decodeURL = (state, url) => [state, new URL(url)] /** * @typedef {{gateway: URL}} EmbedOption @@ -51,10 +53,12 @@ const isObject = (value) => typeof value === 'object' && value != null /** * @param {any} value + * @param {Set} assetPaths + * @param {PropertyKey[]} path * @returns {value is API.EncodedURL} */ -const isEncodedURL = (value) => - value != null && value['@'] === 'URL' && typeof value.href === 'string' +const isEncodedURL = (value, assetPaths, path) => + typeof value === 'string' && assetPaths.has(path.join('.')) /** * Takes token input and encodes it into @@ -129,7 +133,7 @@ const isBlob = (value) => value instanceof Blob * * @template T, I, X, O, State * @param {API.Encoded} input - Arbitrary input. - * @param {(input:any) => input is X} p - Predicate function to determine + * @param {(input:any, state:State, path:PropertyKey[]) => input is X} p - Predicate function to determine * which values to swap. * @param {(state:State, input:X, path:PropertyKey[]) => [State, O]} f - Function * that swaps matching values. @@ -145,7 +149,7 @@ export const mapWith = (input, p, f, state) => { /** * @template T, I, X, O, State * @param {API.Encoded} input - Arbitrary input. - * @param {(input:any) => input is X} p - Predicate function to determine + * @param {(input:any, state:State, path:PropertyKey[]) => input is X} p - Predicate function to determine * which values to swap. * @param {(state:State, input:X, path:PropertyKey[]) => [State, O]} f - Function * that swaps matching values. @@ -155,7 +159,7 @@ export const mapWith = (input, p, f, state) => { * @returns {[State, API.Encoded]} */ const mapValueWith = (input, p, f, state, path) => - p(input) + p(input, state, path) ? f(state, input, path) : Array.isArray(input) ? mapArrayWith(input, p, f, state, path) @@ -168,7 +172,7 @@ const mapValueWith = (input, p, f, state, path) => * * @template State, T, I, X, O * @param {API.Encoded} input - * @param {(input:any) => input is X} p + * @param {(input:any, state:State, path:PropertyKey[]) => input is X} p * @param {(state: State, input:X, path:PropertyKey[]) => [State, O]} f * @param {State} init * @param {PropertyKey[]} path @@ -192,7 +196,7 @@ const mapObjectWith = (input, p, f, init, path) => { * @template I, X, O, State * @template {any[]} T * @param {T} input - * @param {(input:any) => input is X} p + * @param {(input:any, state:State, path:PropertyKey[]) => input is X} p * @param {(state: State, input:X, path:PropertyKey[]) => [State, O]} f * @param {State} init * @param {PropertyKey[]} path diff --git a/client/test/lib.spec.js b/client/test/lib.spec.js index 7c9da69c81..86ddf4484a 100644 --- a/client/test/lib.spec.js +++ b/client/test/lib.spec.js @@ -206,12 +206,15 @@ describe('client', () => { it('store with properties', async () => { const client = new NFTStorage({ token, endpoint }) + const trick = + 'ipfs://bafyreiemweb3jxougg7vaovg7wyiohwqszmgwry5xwitw3heepucg6vyd4' const result = await client.store({ name: 'name', description: 'stuff', image: new File(['fake image'], 'cat.png', { type: 'image/png' }), properties: { extra: 'meta', + trick, src: [ new File(['hello'], 'hello.txt', { type: 'text/plain' }), new Blob(['bye']), @@ -231,6 +234,7 @@ describe('client', () => { assert.ok(result.data.image.protocol, 'ipfs:') assert.equal(result.data.properties.extra, 'meta') + assert.equal(result.data.properties.trick, trick) assert.ok(Array.isArray(result.data.properties.src)) assert.equal(result.data.properties.src.length, 2) diff --git a/client/test/service.js b/client/test/service.js index 269a8ec0d5..546a1308e0 100644 --- a/client/test/service.js +++ b/client/test/service.js @@ -54,7 +54,6 @@ const importToken = async (request) => { const data = JSON.parse(/** @type {string} */ (form.get('meta'))) const dag = JSON.parse(JSON.stringify(data)) - const metadata = JSON.parse(JSON.stringify(data)) for (const [name, content] of form.entries()) { if (name !== 'meta') { @@ -62,15 +61,12 @@ const importToken = async (request) => { const cid = await importAsset(file) const href = `ipfs://${cid}/${file.name}` const path = name.split('.') + setIn(data, path, href) setIn(dag, path, cid) - setIn(data, path, { '@': 'URL', href }) - setIn(metadata, path, href) } } - dag.meta = await importAsset( - new File([JSON.stringify(metadata)], 'data.json') - ) + dag.meta = await importAsset(new File([JSON.stringify(data)], 'data.json')) const bytes = CBOR.encode(dag) const hash = await sha256.digest(bytes) @@ -80,7 +76,7 @@ const importToken = async (request) => { ok: true, value: { ipld: cid.toString(), - metadata: { '@': 'URL', href: `ipfs://${dag.meta}/data.json` }, + metadata: `ipfs://${dag.meta}/data.json`, data, }, } From bca5cf4da0bcafcb1f8965a5fea9bbc58b1dcbbc Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Wed, 7 Apr 2021 16:22:36 -0700 Subject: [PATCH 09/25] chore: refine store api --- client/src/lib.js | 18 ++++--------- client/src/lib/interface.ts | 26 +++++++++---------- client/src/token.js | 51 ++++++++++++++++++++++++++++++++++--- client/test/importer.js | 2 +- client/test/lib.spec.js | 50 +++++++++++++++++++++--------------- client/test/service.js | 15 ++++++----- 6 files changed, 104 insertions(+), 58 deletions(-) diff --git a/client/src/lib.js b/client/src/lib.js index 0c2aedadae..53cf8391e6 100644 --- a/client/src/lib.js +++ b/client/src/lib.js @@ -17,7 +17,6 @@ import * as API from './lib/interface.js' import * as Token from './token.js' import { fetch, File, Blob, FormData } from './platform.js' -const GATEWAY = new URL('https://gateway.ipfs.io/') const ENDPOINT = new URL('https://nft.storage') /** @@ -119,7 +118,7 @@ class NFTStorage { * @template {API.TokenInput} T * @param {API.Service} service * @param {T} data - * @returns {Promise>} + * @returns {Promise>} */ static async store( { endpoint, token }, @@ -168,16 +167,7 @@ class NFTStorage { if (result.ok === true) { const { value } = result - const data = Token.decode(value.data, paths) - - return { - ipld: value.ipld, - metadata: new URL(value.metadata), - data, - embed: Token.embed(data, { - gateway: GATEWAY, - }), - } + return Token.decode(value, paths) } else { throw new Error(result.error.message) } @@ -297,13 +287,15 @@ class NFTStorage { /** * @template {API.TokenInput} T * @param {T} token - * @returns {Promise>} + * @returns {Promise>} */ store(token) { return NFTStorage.store(this, token) } } +const TokenModel = Token.Token +export { TokenModel as Token } export { NFTStorage, File, Blob, FormData } /** diff --git a/client/src/lib/interface.ts b/client/src/lib/interface.ts index 390f017cb1..cc8fd41afa 100644 --- a/client/src/lib/interface.ts +++ b/client/src/lib/interface.ts @@ -34,10 +34,7 @@ export interface API { * `ipfs://bafy...hash/image/cat.png`. For `Blob` object URL will not have * name or mime type instead it will look more like `ipfs://bafy...hash/image/blob` */ - store( - service: Service, - token: T - ): Promise> + store(service: Service, token: T): Promise> /** * Stores a single file and returns a corresponding CID. @@ -199,17 +196,17 @@ interface Localization { locales: string[] } -export interface StoreResult { +export interface Token { /** * CID for the token that encloses all of the files including metadata.json * for the stored token. */ - ipld: CIDString + ipnft: CIDString /** * URL like `ipfs://bafy...hash/meta/data.json` for the stored token metadata. */ - metadata: URL + url: EncodedURL /** * Actual token data in ERC-1155 format. It matches data passed as `token` @@ -223,7 +220,7 @@ export interface StoreResult { * Files/Blobs are substituted with IPFS gateway URLs so they can be * embedded in browsers that do not support `ipfs://` protocol. */ - embed: Encoded + embed(): Encoded } export type EncodedError = { @@ -233,13 +230,14 @@ export type EncodedURL = Tagged export type Result = { ok: true; value: T } | { ok: false; error: X } -export type StoreResponse = Result< +export interface EncodedToken { + ipnft: CIDString + url: EncodedURL + data: Encoded +} +export type StoreResponse = Result< EncodedError, - { - ipld: CIDString - metadata: EncodedURL - data: Encoded - } + EncodedToken > /** diff --git a/client/src/token.js b/client/src/token.js index cc9d5a7516..a1b9f1f2dc 100644 --- a/client/src/token.js +++ b/client/src/token.js @@ -1,6 +1,49 @@ import * as API from './lib/interface.js' import { Blob, FormData } from './platform.js' +const GATEWAY = new URL('https://dweb.link/') + +/** + * @template {API.TokenInput} T + * @implements {API.Token} + */ +export class Token { + /** + * @param {API.CIDString} ipnft + * @param {API.EncodedURL} url + * @param {API.Encoded} data + */ + constructor(ipnft, url, data) { + /** @readonly */ + this.ipnft = ipnft + /** @readonly */ + this.url = url + /** @readonly */ + this.data = data + + Object.defineProperties(this, { + ipnft: { enumerable: true, writable: false }, + url: { enumerable: true, writable: false }, + data: { enumerable: false, writable: false }, + }) + } + /** + * @returns {API.Encoded} + */ + embed() { + return Token.embed(this) + } + + /** + * @template {API.TokenInput} T + * @param {{data: API.Encoded} + */ + static embed({ data }) { + return embed(data, { gateway: GATEWAY }) + } +} + /** * @template T * @param {API.Encoded} input @@ -12,12 +55,12 @@ export const embed = (input, options) => /** * @template {API.TokenInput} T - * @param {API.Encoded} value + * @param {API.EncodedToken} value * @param {Set} paths - Paths were to expcet EncodedURLs - * @returns {API.Encoded} + * @returns {Token} */ -export const decode = (value, paths) => - mapWith(value, isEncodedURL, decodeURL, paths) +export const decode = ({ ipnft, url, data }, paths) => + new Token(ipnft, url, mapWith(data, isEncodedURL, decodeURL, paths)) /** * @param {any} value diff --git a/client/test/importer.js b/client/test/importer.js index b8a826dacc..59765d6cbc 100644 --- a/client/test/importer.js +++ b/client/test/importer.js @@ -56,7 +56,7 @@ class Block { /** * - * @param {Uint8Array} content + * @param {Uint8Array|string} content */ export const importBlob = async (content) => { const results = importer([{ content }], new Block(), { onlyHash: true }) diff --git a/client/test/lib.spec.js b/client/test/lib.spec.js index 86ddf4484a..9c6e3379a8 100644 --- a/client/test/lib.spec.js +++ b/client/test/lib.spec.js @@ -1,7 +1,9 @@ import * as assert from 'uvu/assert' -import { NFTStorage, Blob, File } from 'nft.storage' +import { NFTStorage, Blob, File, Token } from 'nft.storage' import { CID } from 'multiformats' +const DWEB_LINK = 'dweb.link' + describe('client', () => { const { AUTH_TOKEN, SERVICE_ENDPOINT } = process.env const token = AUTH_TOKEN || '' @@ -187,21 +189,22 @@ describe('client', () => { image: new Blob(['fake image'], { type: 'image/png' }), }) - assert.ok(result.metadata instanceof URL) - assert.ok(result.metadata.protocol, 'ipfs:') + assert.ok(typeof result.url === 'string') + assert.ok(new URL(result.url).protocol, 'ipfs:') - const cid = CID.parse(result.ipld) - assert.equal(cid.version, 1) + assert.ok(typeof result.ipnft === 'string') + assert.equal(CID.parse(result.ipnft).version, 1) assert.equal(result.data.name, 'name') assert.equal(result.data.description, 'stuff') assert.ok(result.data.image instanceof URL) assert.ok(result.data.image.protocol, 'ipfs:') - assert.equal(result.embed.name, 'name') - assert.equal(result.embed.description, 'stuff') - assert.ok(result.embed.image instanceof URL) - assert.ok(result.embed.image.protocol, 'https:') + const embed = result.embed() + assert.equal(embed.name, 'name') + assert.equal(embed.description, 'stuff') + assert.ok(embed.image instanceof URL) + assert.ok(embed.image.protocol, 'https:') }) it('store with properties', async () => { @@ -222,12 +225,14 @@ describe('client', () => { }, }) - assert.ok(result.metadata instanceof URL) - assert.ok(result.metadata.protocol, 'ipfs:') + assert.ok(result instanceof Token) - const cid = CID.parse(result.ipld) + const cid = CID.parse(result.ipnft) assert.equal(cid.version, 1) + assert.ok(typeof result.url === 'string') + assert.ok(result.url.startsWith('ipfs:')) + assert.equal(result.data.name, 'name') assert.equal(result.data.description, 'stuff') assert.ok(result.data.image instanceof URL) @@ -245,21 +250,26 @@ describe('client', () => { assert.ok(b instanceof URL) assert.equal(b.protocol, 'ipfs:') - assert.equal(result.embed.name, 'name') - assert.equal(result.embed.description, 'stuff') - assert.ok(result.embed.image instanceof URL) - assert.ok(result.embed.image.protocol, 'https:') + const embed = result.embed() + + assert.equal(embed.name, 'name') + assert.equal(embed.description, 'stuff') + assert.ok(embed.image instanceof URL) + assert.ok(embed.image.protocol, 'https:') + assert.ok(embed.image.host, DWEB_LINK) - assert.equal(result.embed.properties.extra, 'meta') - assert.ok(Array.isArray(result.embed.properties.src)) - assert.equal(result.embed.properties.src.length, 2) + assert.equal(embed.properties.extra, 'meta') + assert.ok(Array.isArray(embed.properties.src)) + assert.equal(embed.properties.src.length, 2) - const [h2, b2] = /** @type {[URL, URL]} */ (result.embed.properties.src) + const [h2, b2] = /** @type {[URL, URL]} */ (embed.properties.src) assert.ok(h2 instanceof URL) assert.equal(h2.protocol, 'https:') + assert.equal(h2.host, DWEB_LINK) assert.ok(b2 instanceof URL) assert.equal(b2.protocol, 'https:') + assert.equal(b2.host, DWEB_LINK) }) }) diff --git a/client/test/service.js b/client/test/service.js index 546a1308e0..4a3134929d 100644 --- a/client/test/service.js +++ b/client/test/service.js @@ -1,5 +1,4 @@ import { CID } from 'multiformats' -import { File } from '../src/platform.js' import { sha256 } from 'multiformats/hashes/sha2' import { importBlob, importDirectory } from './importer.js' import { Response, Request } from './mock-server.js' @@ -66,17 +65,21 @@ const importToken = async (request) => { } } - dag.meta = await importAsset(new File([JSON.stringify(data)], 'data.json')) + const metadata = await importBlob(JSON.stringify(data)) - const bytes = CBOR.encode(dag) + const bytes = CBOR.encode({ + ...dag, + 'metadata.json': metadata.cid, + type: 'nft', + }) const hash = await sha256.digest(bytes) - const cid = CID.create(1, CBOR.code, hash) + const ipnft = CID.create(1, CBOR.code, hash) const result = { ok: true, value: { - ipld: cid.toString(), - metadata: `ipfs://${dag.meta}/data.json`, + ipnft: ipnft.toString(), + url: `ipfs://${ipnft}/metadata.json`, data, }, } From b8ef5db9849fb1064d8825f26b8c2d2ceb65f183 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Thu, 8 Apr 2021 13:56:05 -0700 Subject: [PATCH 10/25] feat: implement backend part --- client/test/importer.js | 1 - site/package.json | 3 +- site/src/bindings.d.ts | 1 + site/src/constants.js | 6 +- site/src/index.js | 4 +- site/src/ipfs.js | 118 ++++++++++++++++++++++++++++++ site/src/routes/nfts-store.js | 83 +++++++++++++++++++++ site/src/utils/form-data.js | 24 ++++++ site/src/utils/multipart/index.js | 11 ++- site/src/utils/utils.js | 27 +++++++ 10 files changed, 272 insertions(+), 6 deletions(-) create mode 100644 site/src/ipfs.js create mode 100644 site/src/routes/nfts-store.js create mode 100644 site/src/utils/form-data.js diff --git a/client/test/importer.js b/client/test/importer.js index 59765d6cbc..1f3dcd149d 100644 --- a/client/test/importer.js +++ b/client/test/importer.js @@ -55,7 +55,6 @@ class Block { } /** - * * @param {Uint8Array|string} content */ export const importBlob = async (content) => { diff --git a/site/package.json b/site/package.json index 9f74e0470f..6821501114 100644 --- a/site/package.json +++ b/site/package.json @@ -18,7 +18,8 @@ "cookie": "^0.4.1", "merge-options": "^3.0.4", "multiformats": "^4.5.3", - "regexparam": "^1.3.0" + "regexparam": "^1.3.0", + "@ipld/dag-cbor": "^4.0.0" }, "devDependencies": { "@cloudflare/workers-types": "^2.1.0", diff --git a/site/src/bindings.d.ts b/site/src/bindings.d.ts index b45bf2b8fd..a690041733 100644 --- a/site/src/bindings.d.ts +++ b/site/src/bindings.d.ts @@ -13,6 +13,7 @@ declare global { const USERS: KVNamespace const NFTS: KVNamespace const PINATA_JWT: string + const IPFS_HOST: string } export interface Pin { diff --git a/site/src/constants.js b/site/src/constants.js index 188db09191..b36aa6c67e 100644 --- a/site/src/constants.js +++ b/site/src/constants.js @@ -1,4 +1,4 @@ -// let AUTH0_DOMAIN, AUTH0_CLIENT_ID, AUTH0_CLIENT_SECRET, SALT, PINATA_JWT +// let AUTH0_DOMAIN, AUTH0_CLIENT_ID, AUTH0_CLIENT_SECRET, SALT, PINATA_JWT, IPFS_HOST export const stores = { auth: SESSION, csrf: CSRF, @@ -19,6 +19,10 @@ export const pinata = { jwt: PINATA_JWT, } +export const ipfs = { + host: IPFS_HOST, +} + export const cookieKey = 'AUTH0-AUTH' export const isDebug = DEBUG === 'true' diff --git a/site/src/index.js b/site/src/index.js index d7e78d3e86..1faa34c66a 100644 --- a/site/src/index.js +++ b/site/src/index.js @@ -5,6 +5,7 @@ import { logout } from './routes/logout.js' import { notFound } from './utils/utils.js' import { cors, postCors } from './routes/cors.js' import { upload } from './routes/nfts-upload.js' +import { store } from './routes/nfts-store.js' import { status } from './routes/nfts-get.js' import { remove } from './routes/nfts-delete.js' import { list } from './routes/nfts-list.js' @@ -26,7 +27,7 @@ import { metrics } from './routes/metrics.js' const r = new Router({ onError(req, err) { return HTTPError.respond(err) - } + }, }) // Site @@ -49,6 +50,7 @@ r.add('post', '/api/pins/:requestid', pinsReplace, [postCors]) r.add('delete', '/api/pins/:requestid', pinsDelete, [postCors]) // Public API r.add('post', '/api/upload', upload, [postCors]) +r.add('post', '/api/store', store, [postCors]) r.add('get', '/api', list, [postCors]) r.add('get', '/api/:cid', status, [postCors]) r.add('delete', '/api/:cid', remove, [postCors]) diff --git a/site/src/ipfs.js b/site/src/ipfs.js new file mode 100644 index 0000000000..353dee7ef6 --- /dev/null +++ b/site/src/ipfs.js @@ -0,0 +1,118 @@ +import * as constants from './constants.js' +import { CID } from 'multiformats' +import { HTTPError } from './errors.js' + +const ENDPOINT = new URL(constants.ipfs.host) + +/** + * @param {RequestInit & { + * path: string + * params?: URLSearchParams | Record + * }} init + */ +const request = async (init) => { + const params = new URLSearchParams(init.params) + const url = new URL(`/api/v0/${init.path}?${params}`, ENDPOINT) + const response = await fetch(url.href, { + method: 'POST', + ...init, + }) + + if (response.ok) { + return await response.json() + } else { + return HTTPError.throw(response.statusText, response.status) + } +} + +/** + * @param {Blob} content + */ +export const importBlob = async (content) => { + const body = new FormData() + body.set('file', content) + + const { Hash: hash } = await request({ + path: 'add', + params: { + pin: 'true', + 'wrap-with-directory': 'false', + 'cid-version': '1', + }, + body, + }) + + return CID.parse(hash) +} + +/** + * @param {File} file + */ +export const importAsset = async (file) => { + const body = new FormData() + body.set('file', file) + + const { Hash: hash } = await request({ + path: 'add', + params: { + pin: 'true', + 'wrap-with-directory': 'true', + 'cid-version': '1', + }, + body, + }) + + return CID.parse(hash) +} + +/** + * @param {Blob} block + */ +export const importBlock = async (block) => { + const body = new FormData() + body.set('file', block) + + const { Key: Hash } = await request({ + path: 'block/put', + params: { + pin: 'true', + format: 'dag-cbor', + mhtype: 'sha2-256', + }, + body, + }) + + return CID.parse(Hash).toV1() +} + +/** + * @param {CID} cid + */ +export const stat = async (cid) => { + const { + Blocks: blocks, + CumulativeSize: cumulativeSize, + Hash: hash, + Local: local, + Size: size, + SizeLocal: sizeLocal, + Type: type, + WithLocality: withLocality, + } = await request({ + path: 'files/stat', + params: { + arg: `/ipfs/${cid}`, + }, + }) + + return { + blocks, + cumulativeSize, + cid: CID.parse(hash).toV1(), + local, + size, + sizeLocal, + type, + withLocality, + } +} diff --git a/site/src/routes/nfts-store.js b/site/src/routes/nfts-store.js new file mode 100644 index 0000000000..0008e9d4a7 --- /dev/null +++ b/site/src/routes/nfts-store.js @@ -0,0 +1,83 @@ +import { HTTPError } from '../errors.js' +import { verifyToken, setIn } from '../utils/utils.js' +import { toFormData } from '../utils/form-data.js' +import * as nfts from '../models/nfts.js' +import { JSONResponse } from '../utils/json-response.js' +import { importAsset, importBlob, importBlock, stat } from '../ipfs.js' +import CBOR from '@ipld/dag-cbor' + +/** + * @typedef {import('../bindings').NFT} NFT + */ + +/** + * @param {FetchEvent} event + */ +export async function store(event) { + const auth = await verifyToken(event) + if (!auth.ok) { + return HTTPError.respond(auth.error) + } + const { user, tokenName } = auth + + const form = await toFormData(event.request) + + const data = JSON.parse(/** @type {string} */ (form.get('meta'))) + const dag = JSON.parse(JSON.stringify(data)) + + const files = [] + + for (const [name, content] of form.entries()) { + if (name !== 'meta') { + const file = /** @type {File} */ (content) + const cid = await importAsset(file) + const href = `ipfs://${cid}/${file.name}` + const path = name.split('.') + setIn(data, path, href) + setIn(dag, path, cid) + files.push({ name: file.name, type: file.type }) + } + } + + const bytes = CBOR.encode({ + ...dag, + 'metadata.json': await importBlob(new Blob([JSON.stringify(data)])), + type: 'nft', + }) + + const ipnft = await importBlock(new Blob([bytes])) + const { cumulativeSize: size } = await stat(ipnft) + const created = new Date().toISOString() + const cid = ipnft.toString() + + /** @type {NFT} */ + const nft = { + cid, + size, + created, + type: 'nft', + scope: tokenName, + files, + pin: { + cid, + size, + status: 'pinned', + created, + }, + } + + await nfts.set({ user, cid }, nft, { + metadata: { pinStatus: 'pinned', size }, + }) + + const result = { + ok: true, + value: { + ipnft: ipnft.toString(), + url: `ipfs://${ipnft}/metadata.json`, + data, + }, + } + + return new JSONResponse(result) +} diff --git a/site/src/utils/form-data.js b/site/src/utils/form-data.js new file mode 100644 index 0000000000..ac624a3d77 --- /dev/null +++ b/site/src/utils/form-data.js @@ -0,0 +1,24 @@ +import { iterateMultipart } from './multipart' + +/** + * @param {Request|Response} source + * @returns {Promise} + */ +export const toFormData = async ({ body, headers }) => { + const contentType = headers.get('Content-Type') || '' + const [type, boundary] = contentType.split(/\s*;\s*boundary=/) + if (type === 'multipart/form-data' && boundary != null && body != null) { + const form = new FormData() + const parts = iterateMultipart(body, boundary) + for await (const { name, data, filename, contentType } of parts) { + if (filename) { + form.append(name, new File([data], filename, { type: contentType })) + } else { + form.append(name, new TextDecoder().decode(data), filename) + } + } + return form + } else { + throw new TypeError('Could not parse content as FormData.') + } +} diff --git a/site/src/utils/multipart/index.js b/site/src/utils/multipart/index.js index a160effccf..3380de8991 100644 --- a/site/src/utils/multipart/index.js +++ b/site/src/utils/multipart/index.js @@ -231,6 +231,13 @@ export async function* streamMultipart(body, boundary) { } } } + +/** + * + * @param {ReadableStream} body + * @param {string} boundary + * @returns {AsyncIterable} + */ export async function* iterateMultipart(body, boundary) { for await (const part of streamMultipart(body, boundary)) { const chunks = [] @@ -248,7 +255,7 @@ export async function* iterateMultipart(body, boundary) { * * @param {ReadableStream | null} body * @param {string} boundary - * @returns {Promise} + * @returns {Promise} */ export async function parseMultipart(body, boundary) { const parts = [] @@ -259,5 +266,5 @@ export async function parseMultipart(body, boundary) { } /** - * @typedef {{name: string, filename: string, contentType: string, data: Uint8Array}[]} FileParts + * @typedef {{name: string, filename: string, contentType: string, data: Uint8Array}} FilePart */ diff --git a/site/src/utils/utils.js b/site/src/utils/utils.js index 7819666131..60e1372c31 100644 --- a/site/src/utils/utils.js +++ b/site/src/utils/utils.js @@ -168,3 +168,30 @@ export async function verifyToken(event, mode = 'both') { return { ok: false, error: new HTTPError('Unauthorized', 401) } } + +/** + * Sets a given `value` at the given `path` on a passed `object`. + * + * @example + * ```js + * const obj = { a: { b: { c: 1 }}} + * setIn(obj, ['a', 'b', 'c'], 5) + * obj.a.b.c //> 5 + * ``` + * + * @template V + * @param {any} object + * @param {string[]} path + * @param {V} value + */ +export const setIn = (object, path, value) => { + const n = path.length - 1 + let target = object + for (let [index, key] of path.entries()) { + if (index === n) { + target[key] = value + } else { + target = target[key] + } + } +} From 8b42d9753e9ceef6cc241adea5e58fbf7df466aa Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Thu, 8 Apr 2021 14:25:08 -0700 Subject: [PATCH 11/25] fix: typo that cause type check to fail --- client/src/token.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/token.js b/client/src/token.js index a1b9f1f2dc..1f2669e1df 100644 --- a/client/src/token.js +++ b/client/src/token.js @@ -36,7 +36,7 @@ export class Token { /** * @template {API.TokenInput} T - * @param {{data: API.Encoded}} token * @returns {API.Encoded} */ static embed({ data }) { From e952844d29dd8470c6091b28b6b06cdc5a6c11f6 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Thu, 8 Apr 2021 14:26:08 -0700 Subject: [PATCH 12/25] fix: regression in pinata.js --- site/src/pinata.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/site/src/pinata.js b/site/src/pinata.js index 77828713a7..f192f63578 100644 --- a/site/src/pinata.js +++ b/site/src/pinata.js @@ -5,7 +5,7 @@ const endpoint = new URL('https://api.pinata.cloud') /** * @typedef {import('./models/users.js').User} User * @typedef {{ok: true, value: {IpfsHash:string, PinSize:number, Timestamp:string}}|{ok:false, error:Response}} PinataResponse - * @typedef {import('./utils/multipart/index.js').FileParts} FileParts + * @typedef {import('./utils/multipart/index.js').FilePart} FilePart */ /** @@ -52,7 +52,7 @@ export const pinFile = async (blob, user) => { } /** - * @param {FileParts} files + * @param {FilePart[]} files * @param {User} user * @returns {Promise} */ @@ -105,8 +105,11 @@ export async function pinFiles(files, user) { * @param {string} cid * @returns {Promise<{ ok: true, value?: any } | { ok: false, error: Response }>} */ -export const pinInfo = async cid => { - const url = new URL(`/data/pinList?status=pinned&hashContains=${encodeURIComponent(cid)}`, endpoint) +export const pinInfo = async (cid) => { + const url = new URL( + `/data/pinList?status=pinned&hashContains=${encodeURIComponent(cid)}`, + endpoint + ) const response = await fetch(url.toString(), { method: 'GET', From c016e0824de7e2e3924001c9b733f202a92af1f9 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Fri, 9 Apr 2021 16:12:23 -0700 Subject: [PATCH 13/25] fix: remaining issues on the backend --- site/src/index.js | 3 +- site/src/ipfs.js | 52 ++++++++++++++++++----------------- site/src/pinata.js | 44 ++++++++++++++++++++++++++++- site/src/routes/nfts-store.js | 18 ++++++++++-- site/src/utils/ndjson.js | 35 +++++++++++++++++++++++ 5 files changed, 123 insertions(+), 29 deletions(-) create mode 100644 site/src/utils/ndjson.js diff --git a/site/src/index.js b/site/src/index.js index 1faa34c66a..7e680f3f42 100644 --- a/site/src/index.js +++ b/site/src/index.js @@ -5,7 +5,7 @@ import { logout } from './routes/logout.js' import { notFound } from './utils/utils.js' import { cors, postCors } from './routes/cors.js' import { upload } from './routes/nfts-upload.js' -import { store } from './routes/nfts-store.js' +import { store, ipfsVersion } from './routes/nfts-store.js' import { status } from './routes/nfts-get.js' import { remove } from './routes/nfts-delete.js' import { list } from './routes/nfts-list.js' @@ -51,6 +51,7 @@ r.add('delete', '/api/pins/:requestid', pinsDelete, [postCors]) // Public API r.add('post', '/api/upload', upload, [postCors]) r.add('post', '/api/store', store, [postCors]) +r.add('get', '/experimental/ipfs/version', ipfsVersion, [postCors]) r.add('get', '/api', list, [postCors]) r.add('get', '/api/:cid', status, [postCors]) r.add('delete', '/api/:cid', remove, [postCors]) diff --git a/site/src/ipfs.js b/site/src/ipfs.js index 353dee7ef6..d023a2a27f 100644 --- a/site/src/ipfs.js +++ b/site/src/ipfs.js @@ -1,8 +1,9 @@ import * as constants from './constants.js' import { CID } from 'multiformats' +import { ndjson } from './utils/ndjson.js' import { HTTPError } from './errors.js' -const ENDPOINT = new URL(constants.ipfs.host) +const ENDPOINT = new URL(`http://${constants.ipfs.host}/`) /** * @param {RequestInit & { @@ -13,13 +14,23 @@ const ENDPOINT = new URL(constants.ipfs.host) const request = async (init) => { const params = new URLSearchParams(init.params) const url = new URL(`/api/v0/${init.path}?${params}`, ENDPOINT) + const response = await fetch(url.href, { method: 'POST', ...init, }) if (response.ok) { - return await response.json() + if (!response.body) { + return [] + } + + const result = [] + for await (const json of ndjson(response.body)) { + result.push(json) + } + + return result } else { return HTTPError.throw(response.statusText, response.status) } @@ -32,7 +43,7 @@ export const importBlob = async (content) => { const body = new FormData() body.set('file', content) - const { Hash: hash } = await request({ + const [{ Hash: hash }] = await request({ path: 'add', params: { pin: 'true', @@ -52,7 +63,7 @@ export const importAsset = async (file) => { const body = new FormData() body.set('file', file) - const { Hash: hash } = await request({ + const [_file, { Hash: hash }] = await request({ path: 'add', params: { pin: 'true', @@ -72,47 +83,38 @@ export const importBlock = async (block) => { const body = new FormData() body.set('file', block) - const { Key: Hash } = await request({ + const [{ Key: hash }] = await request({ path: 'block/put', params: { pin: 'true', - format: 'dag-cbor', + format: 'cbor', mhtype: 'sha2-256', }, body, }) - return CID.parse(Hash).toV1() + return CID.parse(hash).toV1() } /** * @param {CID} cid */ export const stat = async (cid) => { - const { - Blocks: blocks, - CumulativeSize: cumulativeSize, - Hash: hash, - Local: local, - Size: size, - SizeLocal: sizeLocal, - Type: type, - WithLocality: withLocality, - } = await request({ - path: 'files/stat', + const [{ NumBlocks: numBlocks, Size: size }] = await request({ + path: 'dag/stat', params: { arg: `/ipfs/${cid}`, + progress: 'false', }, }) return { - blocks, - cumulativeSize, - cid: CID.parse(hash).toV1(), - local, size, - sizeLocal, - type, - withLocality, + numBlocks, } } + +export const version = async () => + request({ + path: 'version', + }) diff --git a/site/src/pinata.js b/site/src/pinata.js index f192f63578..53d80d0e3b 100644 --- a/site/src/pinata.js +++ b/site/src/pinata.js @@ -1,4 +1,4 @@ -import { pinata } from './constants.js' +import { pinata, ipfs } from './constants.js' const endpoint = new URL('https://api.pinata.cloud') @@ -51,6 +51,48 @@ export const pinFile = async (blob, user) => { } } +/** + * @typedef {{ok: true, value: {ipfsHash:string, id:string, name:string, status:'string'}}|{ok:false, error:Response}} PinResponse + * + * @see https://pinata.cloud/documentation#PinByHash + * @param {import('multiformats').CID} cid + * @param {User} user + * @returns {Promise} + */ +export const pinCID = async (cid, user) => { + const url = new URL('/pinning/pinByHash', endpoint) + + const response = await fetch(url.toString(), { + body: JSON.stringify({ + hashToPin: `${cid}`, + pinataMetadata: { + name: `${user.nickname}-${Date.now()}`, + keyvalues: { + origin: 'https://nft.storage/', + }, + }, + pinataOptions: { + // Hardcoding this isn't great, but seems better than asknig node each + // time. + hostNodes: [ + `/dns4/${ipfs.host}/tcp/4001/p2p/12D3KooWF8wxbXQ4DNpFLzg44Gpb6NdsTHkG4Bn1Z7a4tWS6rrdq`, + `/dns4/${ipfs.host}/udp/4001/quic/p2p/12D3KooWF8wxbXQ4DNpFLzg44Gpb6NdsTHkG4Bn1Z7a4tWS6rrdq`, + ], + }, + }), + method: 'POST', + headers: { + authorization: `Bearer ${pinata.jwt}`, + }, + }) + + if (response.ok) { + return { ok: true, value: await response.json() } + } else { + return { ok: false, error: response } + } +} + /** * @param {FilePart[]} files * @param {User} user diff --git a/site/src/routes/nfts-store.js b/site/src/routes/nfts-store.js index 0008e9d4a7..4bb7a0474a 100644 --- a/site/src/routes/nfts-store.js +++ b/site/src/routes/nfts-store.js @@ -3,8 +3,9 @@ import { verifyToken, setIn } from '../utils/utils.js' import { toFormData } from '../utils/form-data.js' import * as nfts from '../models/nfts.js' import { JSONResponse } from '../utils/json-response.js' -import { importAsset, importBlob, importBlock, stat } from '../ipfs.js' +import { importAsset, importBlob, importBlock, stat, version } from '../ipfs.js' import CBOR from '@ipld/dag-cbor' +import * as pinata from '../pinata.js' /** * @typedef {import('../bindings').NFT} NFT @@ -46,7 +47,13 @@ export async function store(event) { }) const ipnft = await importBlock(new Blob([bytes])) - const { cumulativeSize: size } = await stat(ipnft) + + // Pinata will start looking for the pin but it may fail or take long time + // and we won't know. Unless we have separate task that keeps cheking for + // status of this. + await pinata.pinCID(ipnft, user) + + const { size } = await stat(ipnft) const created = new Date().toISOString() const cid = ipnft.toString() @@ -81,3 +88,10 @@ export async function store(event) { return new JSONResponse(result) } + +/** + * @param {FetchEvent} event + */ +export async function ipfsVersion(event) { + return new JSONResponse(await version()) +} diff --git a/site/src/utils/ndjson.js b/site/src/utils/ndjson.js new file mode 100644 index 0000000000..d48e2f4410 --- /dev/null +++ b/site/src/utils/ndjson.js @@ -0,0 +1,35 @@ +const BR = /\r?\n/ + +/** + * @param {ReadableStream} source + * @returns {AsyncIterable} + */ +export const ndjson = async function* (source) { + const decoder = new TextDecoder() + const reader = source.getReader() + let buffer = '' + let done = false + + try { + while (!done) { + const chunk = await reader.read() + if (chunk.done) { + done = chunk.done + } else { + buffer += decoder.decode(chunk.value, { stream: true }) + const lines = buffer.split(BR) + for (const line of lines) { + const part = line.trim() + if (part.length > 0) { + yield JSON.parse(part) + } + } + } + } + } finally { + if (!done) { + await reader.cancel() + } + reader.releaseLock() + } +} From d21ec4d760d8b025f8db9159c3750c87cd4deb7c Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Mon, 12 Apr 2021 13:45:48 -0700 Subject: [PATCH 14/25] chore: update addresses --- site/src/pinata.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/src/pinata.js b/site/src/pinata.js index 53d80d0e3b..9c59f667ae 100644 --- a/site/src/pinata.js +++ b/site/src/pinata.js @@ -75,8 +75,8 @@ export const pinCID = async (cid, user) => { // Hardcoding this isn't great, but seems better than asknig node each // time. hostNodes: [ - `/dns4/${ipfs.host}/tcp/4001/p2p/12D3KooWF8wxbXQ4DNpFLzg44Gpb6NdsTHkG4Bn1Z7a4tWS6rrdq`, - `/dns4/${ipfs.host}/udp/4001/quic/p2p/12D3KooWF8wxbXQ4DNpFLzg44Gpb6NdsTHkG4Bn1Z7a4tWS6rrdq`, + `/dns4/${ipfs.host}/tcp/24001/p2p/12D3KooWF8wxbXQ4DNpFLzg44Gpb6NdsTHkG4Bn1Z7a4tWS6rrdq`, + `/dns4/${ipfs.host}/udp/24001/quic/p2p/12D3KooWF8wxbXQ4DNpFLzg44Gpb6NdsTHkG4Bn1Z7a4tWS6rrdq`, ], }, }), From 7650543f35321b6abbc1aa68089e27cc3afd5fdd Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Wed, 5 May 2021 11:20:05 -0700 Subject: [PATCH 15/25] chore: update implementation to use a cluster --- client/package.json | 4 +- site/package.json | 7 +- site/src/cluster.js | 42 +++++++++-- site/src/constants.js | 6 +- site/src/index.js | 2 +- site/src/ipfs.js | 120 ------------------------------ site/src/routes/nfts-store.js | 55 +++++++------- site/src/routes/nfts-upload.js | 21 +++--- site/src/utils/car.js | 24 ++++++ site/src/utils/multipart/index.js | 16 +--- site/tsconfig.json | 1 + 11 files changed, 109 insertions(+), 189 deletions(-) delete mode 100644 site/src/ipfs.js create mode 100644 site/src/utils/car.js diff --git a/client/package.json b/client/package.json index d7fff24e99..ab25215fe9 100644 --- a/client/package.json +++ b/client/package.json @@ -50,8 +50,8 @@ "ipld-in-memory": "8.0.0", "mocha": "8.3.2", "multicodec": "3.0.1", - "multiformats": "^4.5.3", - "@ipld/dag-cbor": "^4.0.0", + "multiformats": "^7.0.0", + "@ipld/dag-cbor": "^5.0.0", "multihashing-async": "2.1.2", "nyc": "15.1.0", "playwright-test": "2.1.0", diff --git a/site/package.json b/site/package.json index 2b89cadecf..908b94c625 100644 --- a/site/package.json +++ b/site/package.json @@ -16,11 +16,12 @@ "license": "MIT", "dependencies": { "@magic-sdk/admin": "^1.3.0", - "@nftstorage/ipfs-cluster": "^2.2.1", - "@ipld/dag-cbor": "^4.0.0", + "@nftstorage/ipfs-cluster": "^2.3.0", + "@ipld/dag-cbor": "^5.0.0", + "@ipld/car": "^1.0.1", "debug": "^4.3.1", "merge-options": "^3.0.4", - "multiformats": "^4.5.3", + "multiformats": "^7.0.0", "p-queue": "^6.6.2", "regexparam": "^1.3.0", "toucan-js": "^2.4.1" diff --git a/site/src/cluster.js b/site/src/cluster.js index 574613d436..bf25c05164 100644 --- a/site/src/cluster.js +++ b/site/src/cluster.js @@ -1,5 +1,6 @@ import { Cluster } from '@nftstorage/ipfs-cluster' import { cluster } from './constants.js' +import { CID } from 'multiformats' const client = new Cluster(cluster.apiUrl, { headers: { Authorization: `Basic ${cluster.basicAuthToken}` }, @@ -13,21 +14,46 @@ const client = new Cluster(cluster.apiUrl, { * @param {Blob} data */ export async function add(data) { - return client.add(data, { metadata: { size: data.size.toString() } }) + const { cid, size } = await client.add(data, { + metadata: { size: data.size.toString() }, + }) + return { + cid, + size: Number(size), + } } /** - * @param {import('./utils/multipart/index.js').FilePart[]} fileParts + * @param {File[]} files */ -export async function addDirectory(fileParts) { - const files = fileParts.map( - (fp) => - new File([fp.data], fp.filename || fp.name, { type: fp.contentType }) - ) +export async function addDirectory(files) { const size = files.reduce((total, f) => total + f.size, 0) - return client.addDirectory(files, { metadata: { size: size.toString() } }) + const results = await client.addDirectory(files, { + metadata: { size: size.toString() }, + }) + return results.map((result) => ({ + cid: result.cid, + size: Number(result.size), + })) } +/** + * Adds given file, wrapped in a diretory, to the cluster and + * returns CID of the diretory back. + * + * @param {File} file + * @returns {Promise} + */ +export const importAsset = async (file) => { + const result = await client.addDirectory([file]) + if (result.length !== 2) { + throw new Error( + `Expected response with two entries, but got ${result.length} instead` + ) + } + const [, dir] = result + return CID.parse(dir.cid) +} /** * @param {string} cid */ diff --git a/site/src/constants.js b/site/src/constants.js index 3d23248f7e..9ae487da0f 100644 --- a/site/src/constants.js +++ b/site/src/constants.js @@ -1,4 +1,4 @@ -// let MAGIC_SECRET_KEY, SALT, PINATA_JWT, IPFS_HOST +// let MAGIC_SECRET_KEY, SALT, PINATA_JWT export const stores = { deals: DEALS, users: USERS, @@ -14,10 +14,6 @@ export const secrets = { sentry: SENTRY_DSN, } -export const ipfs = { - host: IPFS_HOST, -} - export const cluster = { apiUrl: CLUSTER_API_URL, basicAuthToken: diff --git a/site/src/index.js b/site/src/index.js index 21b2c7b60f..cdd963c3fa 100644 --- a/site/src/index.js +++ b/site/src/index.js @@ -4,7 +4,7 @@ import { HTTPError } from './errors.js' import { cors, postCors } from './routes/cors.js' import { check } from './routes/nfts-check.js' import { upload } from './routes/nfts-upload.js' -import { store, ipfsVersion } from './routes/nfts-store.js' +import { store } from './routes/nfts-store.js' import { status } from './routes/nfts-get.js' import { remove } from './routes/nfts-delete.js' import { list } from './routes/nfts-list.js' diff --git a/site/src/ipfs.js b/site/src/ipfs.js deleted file mode 100644 index d023a2a27f..0000000000 --- a/site/src/ipfs.js +++ /dev/null @@ -1,120 +0,0 @@ -import * as constants from './constants.js' -import { CID } from 'multiformats' -import { ndjson } from './utils/ndjson.js' -import { HTTPError } from './errors.js' - -const ENDPOINT = new URL(`http://${constants.ipfs.host}/`) - -/** - * @param {RequestInit & { - * path: string - * params?: URLSearchParams | Record - * }} init - */ -const request = async (init) => { - const params = new URLSearchParams(init.params) - const url = new URL(`/api/v0/${init.path}?${params}`, ENDPOINT) - - const response = await fetch(url.href, { - method: 'POST', - ...init, - }) - - if (response.ok) { - if (!response.body) { - return [] - } - - const result = [] - for await (const json of ndjson(response.body)) { - result.push(json) - } - - return result - } else { - return HTTPError.throw(response.statusText, response.status) - } -} - -/** - * @param {Blob} content - */ -export const importBlob = async (content) => { - const body = new FormData() - body.set('file', content) - - const [{ Hash: hash }] = await request({ - path: 'add', - params: { - pin: 'true', - 'wrap-with-directory': 'false', - 'cid-version': '1', - }, - body, - }) - - return CID.parse(hash) -} - -/** - * @param {File} file - */ -export const importAsset = async (file) => { - const body = new FormData() - body.set('file', file) - - const [_file, { Hash: hash }] = await request({ - path: 'add', - params: { - pin: 'true', - 'wrap-with-directory': 'true', - 'cid-version': '1', - }, - body, - }) - - return CID.parse(hash) -} - -/** - * @param {Blob} block - */ -export const importBlock = async (block) => { - const body = new FormData() - body.set('file', block) - - const [{ Key: hash }] = await request({ - path: 'block/put', - params: { - pin: 'true', - format: 'cbor', - mhtype: 'sha2-256', - }, - body, - }) - - return CID.parse(hash).toV1() -} - -/** - * @param {CID} cid - */ -export const stat = async (cid) => { - const [{ NumBlocks: numBlocks, Size: size }] = await request({ - path: 'dag/stat', - params: { - arg: `/ipfs/${cid}`, - progress: 'false', - }, - }) - - return { - size, - numBlocks, - } -} - -export const version = async () => - request({ - path: 'version', - }) diff --git a/site/src/routes/nfts-store.js b/site/src/routes/nfts-store.js index 5b14aca59f..9f4d87cb90 100644 --- a/site/src/routes/nfts-store.js +++ b/site/src/routes/nfts-store.js @@ -1,12 +1,15 @@ -import { HTTPError } from '../errors.js' import { validate } from '../utils/auth.js' import { setIn } from '../utils/utils.js' import { toFormData } from '../utils/form-data.js' import * as nfts from '../models/nfts.js' import { JSONResponse } from '../utils/json-response.js' -import { importAsset, importBlob, importBlock, stat, version } from '../ipfs.js' -import CBOR from '@ipld/dag-cbor' +import * as CBOR from '@ipld/dag-cbor' import * as pinata from '../pinata.js' +import * as cluster from '../cluster.js' +import { CID } from 'multiformats' +import { sha256 } from 'multiformats/hashes/sha2' +import * as Block from 'multiformats/block' +import * as CAR from '../utils/car.js' /** * @typedef {import('../bindings').NFT} NFT @@ -27,7 +30,7 @@ export async function store(event) { for (const [name, content] of form.entries()) { if (name !== 'meta') { const file = /** @type {File} */ (content) - const cid = await importAsset(file) + const cid = await cluster.importAsset(file) const href = `ipfs://${cid}/${file.name}` const path = name.split('.') setIn(data, path, href) @@ -36,22 +39,31 @@ export async function store(event) { } } - const bytes = CBOR.encode({ - ...dag, - 'metadata.json': await importBlob(new Blob([JSON.stringify(data)])), - type: 'nft', + const metadata = await cluster.add(new Blob([JSON.stringify(data)])) + const block = await Block.encode({ + value: { + ...dag, + 'metadata.json': CID.parse(metadata.cid), + type: 'nft', + }, + codec: CBOR, + hasher: sha256, }) + const car = await CAR.encode([block.cid], [block]) + const { cid, size } = await cluster.add(car) - const ipnft = await importBlock(new Blob([bytes])) + // We do want worker to wait for this, but we do not want to + // block response waiting on this. + event.waitUntil( + pinata + .pinByHash(cid, { + pinataOptions: { hostNodes: cluster.delegates() }, + pinataMetadata: { name: `${user.nickname}-${Date.now()}` }, + }) + .catch((error) => console.error(error)) + ) - // Pinata will start looking for the pin but it may fail or take long time - // and we won't know. Unless we have separate task that keeps cheking for - // status of this. - await pinata.pinCID(ipnft, user) - - const { size } = await stat(ipnft) const created = new Date().toISOString() - const cid = ipnft.toString() /** @type {NFT} */ const nft = { @@ -76,18 +88,11 @@ export async function store(event) { const result = { ok: true, value: { - ipnft: ipnft.toString(), - url: `ipfs://${ipnft}/metadata.json`, + ipnft: cid, + url: `ipfs://${cid}/metadata.json`, data, }, } return new JSONResponse(result) } - -/** - * @param {FetchEvent} event - */ -export async function ipfsVersion(event) { - return new JSONResponse(await version()) -} diff --git a/site/src/routes/nfts-upload.js b/site/src/routes/nfts-upload.js index 480b822251..baf1fc4ae6 100644 --- a/site/src/routes/nfts-upload.js +++ b/site/src/routes/nfts-upload.js @@ -1,5 +1,5 @@ import { HTTPError } from '../errors.js' -import { parseMultipart } from '../utils/multipart/index.js' +import { toFormData } from '../utils/form-data.js' import * as pinata from '../pinata.js' import * as cluster from '../cluster.js' import * as nfts from '../models/nfts.js' @@ -19,9 +19,12 @@ export async function upload(event) { const { user, tokenName } = await validate(event) if (contentType.includes('multipart/form-data')) { - const boundary = contentType.split('boundary=')[1].trim() - const parts = await parseMultipart(event.request.body, boundary) - const dir = await cluster.addDirectory(parts) + const form = await toFormData(event.request) + // Our API schema requires that all file parts be named `file` and + // encoded as binary, which is why we can expect that each part here is + // a file (and not a stirng). + const files = /** @type {File[]} */ (form.getAll('file')) + const dir = await cluster.addDirectory(files) const { cid, size } = dir[dir.length - 1] event.waitUntil( (async () => { @@ -39,18 +42,16 @@ export async function upload(event) { /** @type {NFT} */ const nft = { cid, - // @ts-ignore - size, + size: size, created: created.toISOString(), type: 'directory', scope: tokenName, - files: parts.map((f) => ({ - name: f.filename || f.name, - type: f.contentType, + files: files.map((f) => ({ + name: f.name, + type: f.type, })), pin: { cid, - // @ts-ignore size, status: 'pinned', created: created.toISOString(), diff --git a/site/src/utils/car.js b/site/src/utils/car.js new file mode 100644 index 0000000000..d16b7b2585 --- /dev/null +++ b/site/src/utils/car.js @@ -0,0 +1,24 @@ +import { CID } from 'multiformats' +import { CarWriter } from '@ipld/car' + +/** + * @param {CID[]} roots + * @param {AsyncIterable|Iterable} blocks + * @returns {Promise} + */ +export const encode = async (roots, blocks) => { + const { out, writer } = CarWriter.create(roots) + for await (const block of blocks) { + writer.put(block) + } + writer.close() + + const parts = [] + for await (const part of out) { + parts.push(part) + } + + return /** @type {Blob & {type: 'application/car'}} */ (new Blob(parts, { + type: 'application/car', + })) +} diff --git a/site/src/utils/multipart/index.js b/site/src/utils/multipart/index.js index 3380de8991..090c8c3b1e 100644 --- a/site/src/utils/multipart/index.js +++ b/site/src/utils/multipart/index.js @@ -252,19 +252,5 @@ export async function* iterateMultipart(body, boundary) { } /** - * - * @param {ReadableStream | null} body - * @param {string} boundary - * @returns {Promise} - */ -export async function parseMultipart(body, boundary) { - const parts = [] - for await (const part of iterateMultipart(body, boundary)) { - parts.push(part) - } - return parts -} - -/** - * @typedef {{name: string, filename: string, contentType: string, data: Uint8Array}} FilePart + * @typedef {{name: string, filename?: string, contentType?: string, data: Uint8Array}} FilePart */ diff --git a/site/tsconfig.json b/site/tsconfig.json index 16e65843c7..d81f097285 100644 --- a/site/tsconfig.json +++ b/site/tsconfig.json @@ -9,6 +9,7 @@ "moduleResolution": "node", "sourceMap": true, "esModuleInterop": true, + "skipLibCheck": true, "noEmit": true, "resolveJsonModule": true, "types": ["@cloudflare/workers-types"] From 407ec0f13e1222bdd3b5cf2bf2a3838a66411df0 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Wed, 5 May 2021 11:20:54 -0700 Subject: [PATCH 16/25] chore: remove redundunt code --- client/test/lib.spec.js | 13 ++++++++---- site/src/pinata.js | 44 +---------------------------------------- 2 files changed, 10 insertions(+), 47 deletions(-) diff --git a/client/test/lib.spec.js b/client/test/lib.spec.js index 60b8d22f38..a41691a060 100644 --- a/client/test/lib.spec.js +++ b/client/test/lib.spec.js @@ -9,6 +9,8 @@ describe('client', () => { const token = AUTH_TOKEN || '' const endpoint = new URL(SERVICE_ENDPOINT || '') + console.log(token) + it('interface', () => { assert.equal(typeof NFTStorage, 'function') const client = new NFTStorage({ token: 'secret' }) @@ -114,7 +116,7 @@ describe('client', () => { }) }) - describe('store', async () => { + describe.only('store', async () => { it('requires name', async () => { const client = new NFTStorage({ token, endpoint }) try { @@ -187,7 +189,7 @@ describe('client', () => { } }) - it('errors without token', async () => { + it.skip('errors without token', async () => { const client = new NFTStorage({ token: 'wrong', endpoint }) try { @@ -217,6 +219,8 @@ describe('client', () => { assert.ok(typeof result.ipnft === 'string') assert.equal(CID.parse(result.ipnft).version, 1) + console.log(result) + assert.equal(result.data.name, 'name') assert.equal(result.data.description, 'stuff') assert.ok(result.data.image instanceof URL) @@ -227,7 +231,7 @@ describe('client', () => { assert.equal(embed.description, 'stuff') assert.ok(embed.image instanceof URL) assert.ok(embed.image.protocol, 'https:') - }) + }).timeout(5000) it('store with properties', async () => { const client = new NFTStorage({ token, endpoint }) @@ -248,6 +252,7 @@ describe('client', () => { }) assert.ok(result instanceof Token) + console.log(result) const cid = CID.parse(result.ipnft) assert.equal(cid.version, 1) @@ -292,7 +297,7 @@ describe('client', () => { assert.ok(b2 instanceof URL) assert.equal(b2.protocol, 'https:') assert.equal(b2.host, DWEB_LINK) - }) + }).timeout(9000) }) describe('status', () => { diff --git a/site/src/pinata.js b/site/src/pinata.js index 7687453df7..aaa93d0540 100644 --- a/site/src/pinata.js +++ b/site/src/pinata.js @@ -1,4 +1,4 @@ -import { secrets, ipfs } from './constants.js' +import { secrets } from './constants.js' const endpoint = new URL('https://api.pinata.cloud') @@ -51,48 +51,6 @@ export const pinFile = async (blob, user) => { } } -/** - * @typedef {{ok: true, value: {ipfsHash:string, id:string, name:string, status:'string'}}|{ok:false, error:Response}} PinResponse - * - * @see https://pinata.cloud/documentation#PinByHash - * @param {import('multiformats').CID} cid - * @param {User} user - * @returns {Promise} - */ -export const pinCID = async (cid, user) => { - const url = new URL('/pinning/pinByHash', endpoint) - - const response = await fetch(url.toString(), { - body: JSON.stringify({ - hashToPin: `${cid}`, - pinataMetadata: { - name: `${user.nickname}-${Date.now()}`, - keyvalues: { - origin: 'https://nft.storage/', - }, - }, - pinataOptions: { - // Hardcoding this isn't great, but seems better than asknig node each - // time. - hostNodes: [ - `/dns4/${ipfs.host}/tcp/24001/p2p/12D3KooWF8wxbXQ4DNpFLzg44Gpb6NdsTHkG4Bn1Z7a4tWS6rrdq`, - `/dns4/${ipfs.host}/udp/24001/quic/p2p/12D3KooWF8wxbXQ4DNpFLzg44Gpb6NdsTHkG4Bn1Z7a4tWS6rrdq`, - ], - }, - }), - method: 'POST', - headers: { - authorization: `Bearer ${secrets.pinata}`, - }, - }) - - if (response.ok) { - return { ok: true, value: await response.json() } - } else { - return { ok: false, error: response } - } -} - /** * @param {FilePart[]} files * @param {User} user From a5413b903933e36404595bbc16e624f3068fd478 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Wed, 5 May 2021 11:23:26 -0700 Subject: [PATCH 17/25] chore: undo unintended test changes --- client/test/lib.spec.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/client/test/lib.spec.js b/client/test/lib.spec.js index a41691a060..97b7d283b3 100644 --- a/client/test/lib.spec.js +++ b/client/test/lib.spec.js @@ -116,7 +116,7 @@ describe('client', () => { }) }) - describe.only('store', async () => { + describe('store', async () => { it('requires name', async () => { const client = new NFTStorage({ token, endpoint }) try { @@ -189,7 +189,7 @@ describe('client', () => { } }) - it.skip('errors without token', async () => { + it('errors without token', async () => { const client = new NFTStorage({ token: 'wrong', endpoint }) try { @@ -231,7 +231,7 @@ describe('client', () => { assert.equal(embed.description, 'stuff') assert.ok(embed.image instanceof URL) assert.ok(embed.image.protocol, 'https:') - }).timeout(5000) + }) it('store with properties', async () => { const client = new NFTStorage({ token, endpoint }) @@ -252,7 +252,6 @@ describe('client', () => { }) assert.ok(result instanceof Token) - console.log(result) const cid = CID.parse(result.ipnft) assert.equal(cid.version, 1) @@ -297,7 +296,7 @@ describe('client', () => { assert.ok(b2 instanceof URL) assert.equal(b2.protocol, 'https:') assert.equal(b2.host, DWEB_LINK) - }).timeout(9000) + }) }) describe('status', () => { From bee00e58a0285115ee87730958e1f5a65040fd4f Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Wed, 5 May 2021 11:25:26 -0700 Subject: [PATCH 18/25] fix: import --- client/test/service.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/test/service.js b/client/test/service.js index ea32404595..1a18801227 100644 --- a/client/test/service.js +++ b/client/test/service.js @@ -2,7 +2,7 @@ import { CID } from 'multiformats' import { sha256 } from 'multiformats/hashes/sha2' import { importBlob, importDirectory } from './importer.js' import { Response, Request } from './mock-server.js' -import CBOR from '@ipld/dag-cbor' +import * as CBOR from '@ipld/dag-cbor' /** * @param {Request} request */ From 4232bd126b6941c4d60bbab7a2c9373f9e791322 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Wed, 5 May 2021 11:32:06 -0700 Subject: [PATCH 19/25] fix: leftovers from previous iteration --- client/test/lib.spec.js | 4 ---- site/src/bindings.d.ts | 1 - site/src/utils/ndjson.js | 35 ----------------------------------- 3 files changed, 40 deletions(-) delete mode 100644 site/src/utils/ndjson.js diff --git a/client/test/lib.spec.js b/client/test/lib.spec.js index 97b7d283b3..60b8d22f38 100644 --- a/client/test/lib.spec.js +++ b/client/test/lib.spec.js @@ -9,8 +9,6 @@ describe('client', () => { const token = AUTH_TOKEN || '' const endpoint = new URL(SERVICE_ENDPOINT || '') - console.log(token) - it('interface', () => { assert.equal(typeof NFTStorage, 'function') const client = new NFTStorage({ token: 'secret' }) @@ -219,8 +217,6 @@ describe('client', () => { assert.ok(typeof result.ipnft === 'string') assert.equal(CID.parse(result.ipnft).version, 1) - console.log(result) - assert.equal(result.data.name, 'name') assert.equal(result.data.description, 'stuff') assert.ok(result.data.image instanceof URL) diff --git a/site/src/bindings.d.ts b/site/src/bindings.d.ts index a1c636e15c..15a2d3d4b9 100644 --- a/site/src/bindings.d.ts +++ b/site/src/bindings.d.ts @@ -12,7 +12,6 @@ declare global { const NFTS_IDX: KVNamespace const METRICS: KVNamespace const PINATA_JWT: string - const IPFS_HOST: string const CLUSTER_API_URL: string const CLUSTER_BASIC_AUTH_TOKEN: string const CLUSTER_IPFS_PROXY_API_URL: string diff --git a/site/src/utils/ndjson.js b/site/src/utils/ndjson.js deleted file mode 100644 index d48e2f4410..0000000000 --- a/site/src/utils/ndjson.js +++ /dev/null @@ -1,35 +0,0 @@ -const BR = /\r?\n/ - -/** - * @param {ReadableStream} source - * @returns {AsyncIterable} - */ -export const ndjson = async function* (source) { - const decoder = new TextDecoder() - const reader = source.getReader() - let buffer = '' - let done = false - - try { - while (!done) { - const chunk = await reader.read() - if (chunk.done) { - done = chunk.done - } else { - buffer += decoder.decode(chunk.value, { stream: true }) - const lines = buffer.split(BR) - for (const line of lines) { - const part = line.trim() - if (part.length > 0) { - yield JSON.parse(part) - } - } - } - } - } finally { - if (!done) { - await reader.cancel() - } - reader.releaseLock() - } -} From 746e7054947c9b3c26c6eed6e06276fa9f1d0bbe Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Thu, 6 May 2021 10:29:55 -0700 Subject: [PATCH 20/25] Apply suggestions from code review Co-authored-by: Alan Shaw --- client/src/lib/interface.ts | 44 +++++++++++++++++++------------------ client/test/lib.spec.js | 2 +- site/src/cluster.js | 2 +- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/client/src/lib/interface.ts b/client/src/lib/interface.ts index 2bbc74b73a..0af95e1d24 100644 --- a/client/src/lib/interface.ts +++ b/client/src/lib/interface.ts @@ -23,20 +23,22 @@ export type CIDString = Tagged export interface API { /** - * Stores given token and all the resources (in form of File or a Blob) it - * references along with a metadata JSON as specificed in (ERC-1155). The - * `token.image` must be either `File` or a `Blob` instance, which will be - * stored and corresponding content address URL will be saved in metadata - * JSON file under `image` field. + * Stores the given token and all resources it references (in the form of a + * File or a Blob) along with a metadata JSON as specificed in ERC-1155. The + * `token.image` must be either a `File` or a `Blob` instance, which will be + * stored and the corresponding content address URL will be saved in the + * metadata JSON file under `image` field. * - * If `token.properties` contain properties with `File` or `Blob` values those - * also get stored and their URLs will be saved in metadata json in their - * place. + * If `token.properties` contains properties with `File` or `Blob` values, + * those also get stored and their URLs will be saved in the metadata JSON + * file in their place. * - * Note: URLs for `File` object will retain the name e.g. in case of - * `new File([bytes], 'cat.png', { type: 'image/png' })` it will look like - * `ipfs://bafy...hash/image/cat.png`. For `Blob` object URL will not have - * name or mime type instead it will look more like `ipfs://bafy...hash/image/blob` + * Note: URLs for `File` objects will retain file names e.g. in case of + * `new File([bytes], 'cat.png', { type: 'image/png' })` will be transformed + * into a URL that looks like `ipfs://bafy...hash/image/cat.png`. For `Blob` + * objects, the URL will not have a file name name or mime type, instead it + * will be transformed into a URL that looks like + * `ipfs://bafy...hash/image/blob`. */ store(service: Service, token: T): Promise> @@ -179,13 +181,13 @@ export interface TokenInput { */ description: string /** - * An `File` with mime type image/* representing the asset this - * token represents. Consider making any images at a width between `320` and + * A `File` with mime type `image/*` representing the asset this + * token represents. Consider creating images with width between `320` and * `1080` pixels and aspect ratio between `1.91:1` and `4:5` inclusive. * - * If `File` object is used, URL in the metadata will include a filename - * e.g. `ipfs://bafy...hash/cat.png`. If `Blob` is used URL in the metadata - * will not include filename or extension e.g. `ipfs://bafy...img/` + * If a `File` object is used, the URL in the metadata will include a filename + * e.g. `ipfs://bafy...hash/cat.png`. If a `Blob` is used, the URL in the + * metadata will not include filename or extension e.g. `ipfs://bafy...img/` */ image: Blob | File @@ -197,10 +199,10 @@ export interface TokenInput { decimals?: number /** - * Arbitrary properties. Values may be strings, numbers, nested object or - * arrays of values. It is possible to provide a `File` or a `Blob` instance - * as property value, in which case it is stored on IPFS and metadata will - * contain URL to it in form of `ipfs://bafy...hash/name.png` or + * Arbitrary properties. Values may be strings, numbers, nested objects or + * arrays of values. It is possible to provide `File` or `Blob` instances + * as property values, which will be stored on IPFS, and metadata will + * contain URLs to them in form of `ipfs://bafy...hash/name.png` or * `ipfs://bafy...file/` respectively. */ properties?: Object diff --git a/client/test/lib.spec.js b/client/test/lib.spec.js index 60b8d22f38..5612991f90 100644 --- a/client/test/lib.spec.js +++ b/client/test/lib.spec.js @@ -120,7 +120,7 @@ describe('client', () => { try { // @ts-expect-error await client.store({}) - assert.unreachable('sholud have failed') + assert.unreachable('should have failed') } catch (error) { assert.ok(error instanceof TypeError) assert.match( diff --git a/site/src/cluster.js b/site/src/cluster.js index bf25c05164..15e8bdb520 100644 --- a/site/src/cluster.js +++ b/site/src/cluster.js @@ -39,7 +39,7 @@ export async function addDirectory(files) { /** * Adds given file, wrapped in a diretory, to the cluster and - * returns CID of the diretory back. + * returns CID of the directory back. * * @param {File} file * @returns {Promise} From ede8e94f72b4e65e5fd4c2e5929feb96b4aa2dce Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Thu, 6 May 2021 10:40:02 -0700 Subject: [PATCH 21/25] chore: switch to just-safe-set --- client/package.json | 3 ++- client/test/service.js | 28 +--------------------------- site/package.json | 3 ++- site/src/routes/nfts-store.js | 2 +- site/src/utils/utils.js | 27 --------------------------- 5 files changed, 6 insertions(+), 57 deletions(-) diff --git a/client/package.json b/client/package.json index ab25215fe9..d2040e6000 100644 --- a/client/package.json +++ b/client/package.json @@ -58,7 +58,8 @@ "rollup": "2.22.1", "rollup-plugin-multi-input": "1.1.1", "typedoc": "0.20.36", - "uvu": "0.5.1" + "uvu": "0.5.1", + "just-safe-set": "^2.2.1" }, "homepage": "https://github.com/ipfs-shipyard/nft.storage/tree/main/client", "bugs": "https://github.com/ipfs-shipyard/nft.storage/issues" diff --git a/client/test/service.js b/client/test/service.js index 1a18801227..d38d7946e2 100644 --- a/client/test/service.js +++ b/client/test/service.js @@ -3,6 +3,7 @@ import { sha256 } from 'multiformats/hashes/sha2' import { importBlob, importDirectory } from './importer.js' import { Response, Request } from './mock-server.js' import * as CBOR from '@ipld/dag-cbor' +import setIn from 'just-safe-set' /** * @param {Request} request */ @@ -90,33 +91,6 @@ const importToken = async (request) => { } } -/** - * Sets a given `value` at the given `path` on a passed `object`. - * - * @example - * ```js - * const obj = { a: { b: { c: 1 }}} - * setIn(obj, ['a', 'b', 'c'], 5) - * obj.a.b.c //> 5 - * ``` - * - * @template V - * @param {any} object - * @param {string[]} path - * @param {V} value - */ -const setIn = (object, path, value) => { - const n = path.length - 1 - let target = object - for (let [index, key] of path.entries()) { - if (index === n) { - target[key] = value - } else { - target = target[key] - } - } -} - /** * @typedef {{AUTH_TOKEN:string, store: Map}} State * @param {string} [token] diff --git a/site/package.json b/site/package.json index 908b94c625..b490d61752 100644 --- a/site/package.json +++ b/site/package.json @@ -24,7 +24,8 @@ "multiformats": "^7.0.0", "p-queue": "^6.6.2", "regexparam": "^1.3.0", - "toucan-js": "^2.4.1" + "toucan-js": "^2.4.1", + "just-safe-set": "^2.2.1" }, "devDependencies": { "@cloudflare/workers-types": "^2.1.0", diff --git a/site/src/routes/nfts-store.js b/site/src/routes/nfts-store.js index 9f4d87cb90..c7263fc5f2 100644 --- a/site/src/routes/nfts-store.js +++ b/site/src/routes/nfts-store.js @@ -1,5 +1,5 @@ import { validate } from '../utils/auth.js' -import { setIn } from '../utils/utils.js' +import setIn from 'just-safe-set' import { toFormData } from '../utils/form-data.js' import * as nfts from '../models/nfts.js' import { JSONResponse } from '../utils/json-response.js' diff --git a/site/src/utils/utils.js b/site/src/utils/utils.js index 95776af5f5..c528e3bd49 100644 --- a/site/src/utils/utils.js +++ b/site/src/utils/utils.js @@ -25,30 +25,3 @@ export async function timed(fn, label, ctx) { sentry.captureException(err) } } - -/** - * Sets a given `value` at the given `path` on a passed `object`. - * - * @example - * ```js - * const obj = { a: { b: { c: 1 }}} - * setIn(obj, ['a', 'b', 'c'], 5) - * obj.a.b.c //> 5 - * ``` - * - * @template V - * @param {any} object - * @param {string[]} path - * @param {V} value - */ -export const setIn = (object, path, value) => { - const n = path.length - 1 - let target = object - for (let [index, key] of path.entries()) { - if (index === n) { - target[key] = value - } else { - target = target[key] - } - } -} From 9eaeffc98f3e7d4560753b3ac0c94a55200c7807 Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Thu, 6 May 2021 14:40:36 -0700 Subject: [PATCH 22/25] chore: Per review feedback remove CID dep --- site/src/cluster.js | 5 ++--- site/src/routes/nfts-store.js | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/site/src/cluster.js b/site/src/cluster.js index 15e8bdb520..e61916fe55 100644 --- a/site/src/cluster.js +++ b/site/src/cluster.js @@ -1,6 +1,5 @@ import { Cluster } from '@nftstorage/ipfs-cluster' import { cluster } from './constants.js' -import { CID } from 'multiformats' const client = new Cluster(cluster.apiUrl, { headers: { Authorization: `Basic ${cluster.basicAuthToken}` }, @@ -42,7 +41,7 @@ export async function addDirectory(files) { * returns CID of the directory back. * * @param {File} file - * @returns {Promise} + * @returns {Promise} */ export const importAsset = async (file) => { const result = await client.addDirectory([file]) @@ -52,7 +51,7 @@ export const importAsset = async (file) => { ) } const [, dir] = result - return CID.parse(dir.cid) + return dir.cid } /** * @param {string} cid diff --git a/site/src/routes/nfts-store.js b/site/src/routes/nfts-store.js index c7263fc5f2..cdf691b2c6 100644 --- a/site/src/routes/nfts-store.js +++ b/site/src/routes/nfts-store.js @@ -30,7 +30,7 @@ export async function store(event) { for (const [name, content] of form.entries()) { if (name !== 'meta') { const file = /** @type {File} */ (content) - const cid = await cluster.importAsset(file) + const cid = CID.parse(await cluster.importAsset(file)) const href = `ipfs://${cid}/${file.name}` const path = name.split('.') setIn(data, path, href) From ffaa115d7d6cb2ad9d560546f9498c45aa19e8bc Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Thu, 6 May 2021 14:40:57 -0700 Subject: [PATCH 23/25] fix: pinByHash by providing content-type --- site/src/pinata.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/site/src/pinata.js b/site/src/pinata.js index aaa93d0540..5070924c3a 100644 --- a/site/src/pinata.js +++ b/site/src/pinata.js @@ -132,11 +132,14 @@ export const pinInfo = async (cid) => { * @returns {Promise<{ ok: true, value: { id: string, ipfsHash: string, status: string, name: string} }|{ ok: false, error: Response }>} */ export async function pinByHash(cid, options) { - const url = new URL('pinning/pinByHash', endpoint) + const url = new URL('/pinning/pinByHash', endpoint) const response = await fetch(url.toString(), { method: 'POST', - headers: { Authorization: `Bearer ${secrets.pinata}` }, + headers: { + Authorization: `Bearer ${secrets.pinata}`, + 'Content-Type': 'application/json', + }, body: JSON.stringify({ hashToPin: cid, ...(options || {}) }), }) From 09500eed2dbda8835ce9df9d5877daa7b4904cbd Mon Sep 17 00:00:00 2001 From: Irakli Gozalishvili Date: Thu, 6 May 2021 14:46:52 -0700 Subject: [PATCH 24/25] chore: add comment to why skipLibCheck is enabled --- site/tsconfig.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/site/tsconfig.json b/site/tsconfig.json index d81f097285..e89b8b66e0 100644 --- a/site/tsconfig.json +++ b/site/tsconfig.json @@ -9,6 +9,9 @@ "moduleResolution": "node", "sourceMap": true, "esModuleInterop": true, + // Need to disable this because generated typedefs omit generics which + // creates a problew for multiformats + // @see https://github.com/multiformats/js-multiformats#typescript-support "skipLibCheck": true, "noEmit": true, "resolveJsonModule": true, From 229deedb068f474b4d39d8a1aa2fefea3b348050 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 7 May 2021 13:42:53 +0100 Subject: [PATCH 25/25] fix: use latest multiformats to fix ts lint err --- site/package.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/site/package.json b/site/package.json index b490d61752..6b144e03a4 100644 --- a/site/package.json +++ b/site/package.json @@ -15,17 +15,17 @@ "author": "Hugo Dias (hugodias.me)", "license": "MIT", "dependencies": { + "@ipld/car": "^1.0.1", + "@ipld/dag-cbor": "^5.0.0", "@magic-sdk/admin": "^1.3.0", "@nftstorage/ipfs-cluster": "^2.3.0", - "@ipld/dag-cbor": "^5.0.0", - "@ipld/car": "^1.0.1", "debug": "^4.3.1", + "just-safe-set": "^2.2.1", "merge-options": "^3.0.4", - "multiformats": "^7.0.0", + "multiformats": "^8.0.3", "p-queue": "^6.6.2", "regexparam": "^1.3.0", - "toucan-js": "^2.4.1", - "just-safe-set": "^2.2.1" + "toucan-js": "^2.4.1" }, "devDependencies": { "@cloudflare/workers-types": "^2.1.0",