feat: Develop and implement innovative methods for managing the IPFS with Helia

packages/app/package.json

@@ -4,12 +4,34 @@
   "private": true,
   "license": "LGPL-2.1",
   "homepage": "https://onposter.github.io",
+  "type": "module",
   "dependencies": {
+    "@chainsafe/libp2p-gossipsub": "^11.0.0",
+    "@chainsafe/libp2p-noise": "^14.0.0",
+    "@chainsafe/libp2p-yamux": "^6.0.1",
     "@chainsafe/web3-context": "^1.3.1",
     "@emotion/babel-plugin": "^11.11.0",
     "@emotion/react": "^11.8.2",
     "@emotion/styled": "^11.8.1",
+    "@helia/delegated-routing-v1-http-api-client": "^1.1.0",
+    "@helia/remote-pinning": "^1.1.1",
+    "@helia/unixfs": "^1.4.3",
     "@hookform/resolvers": "^2.8.8",
+    "@ipfs-shipyard/pinning-service-client": "^1.0.3",
+    "@libp2p/autonat": "^1.0.1",
+    "@libp2p/bootstrap": "^10.0.2",
+    "@libp2p/circuit-relay-v2": "^1.0.2",
+    "@libp2p/dcutr": "^1.0.1",
+    "@libp2p/identify": "1.0.1",
+    "@libp2p/interface": "^1.0.1",
+    "@libp2p/ipni-content-routing": "^2.0.2",
+    "@libp2p/kad-dht": "^11.0.2",
+    "@libp2p/keychain": "^4.0.2",
+    "@libp2p/mplex": "^10.0.2",
+    "@libp2p/ping": "^1.0.1",
+    "@libp2p/webrtc": "^4.0.3",
+    "@libp2p/websockets": "^8.0.2",
+    "@libp2p/webtransport": "^4.0.3",
     "@mui/icons-material": "^5.5.1",
     "@mui/material": "^5.5.0",
     "@mui/styles": "^5.5.0",
@@ -23,21 +45,28 @@
     "add": "^2.0.6",
     "axios": "^0.26.1",
     "blockies-ts": "^1.0.0",
+    "blockstore-core": "^4.0.0",
+    "blockstore-level": "^1.1.3",
     "browserslist-to-esbuild": "^1.2.0",
+    "datastore-core": "^9.0.0",
     "draft-convert": "^2.1.13",
     "draft-js": "^0.11.7",
     "draft-js-export-html": "^1.4.1",
     "draft-js-import-html": "^1.4.1",
     "ethers": "^5.7.2",
     "gh-pages": "^3.2.3",
     "graphql": "^16.3.0",
+    "helia": "^2.0.0",
     "ipfs-core": "0.14.3",
     "ipfs-http-client": "54.0.0",
+    "ipns": "^7.0.1",
     "is-ipfs": "^6.0.2",
+    "libp2p": "^1.0.3",
     "lodash": "^4.17.21",
     "markdown-to-jsx": "^7.1.7",
     "marked": "^5.1.2",
     "moment": "^2.29.2",
+    "multiformats": "^12.0.1",
     "notistack": "^2.0.8",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",

packages/app/src/App.tsx

@@ -1,4 +1,4 @@
-import React, { useEffect, useState } from "react"
+import React, { useCallback, useEffect, useState } from "react"
 import { PublicationView } from "./components/views/publication/PublicationView"
 import { Routes, Route } from "react-router-dom"
 import { SnackbarProvider } from "notistack"
@@ -19,13 +19,15 @@ import { WalletProvider } from "./connectors/WalletProvider"
 import { RedirectOldRoute } from "./components/commons/RedicrectOldRoute"
 import PreviewArticleView from "./components/views/publication/PreviewArticleView"
 import { EnsProvider } from "./services/ens/context"
+import { useIPFSContext } from "./services/ipfs/context"
 
 const App: React.FC = () => {
   // the chainId should be from the publication if its present
 
   const { chainId: initialChainIdFromProvider } = useWeb3React() // chain id from connected wallet
   const [chainId, setChainId] = useState(initialChainIdFromProvider)
   const [currentSubgraphClient, setCurrentSubgraphClient] = useState(subgraphClient(chainId))
+  const { helia, startHelia, startingHelia, decodeCID } = useIPFSContext()
 
   const updateChainId = (newChainId: number | undefined) => {
     if (newChainId !== chainId) {
@@ -41,6 +43,16 @@ const App: React.FC = () => {
     setChainId(initialChainIdFromProvider)
   }, [initialChainIdFromProvider])
 
+  const initiateHelia = useCallback(async () => {
+    if (!helia && !startingHelia) {
+      await startHelia()
+    }
+  }, [helia, startingHelia, startHelia])
+
+  useEffect(() => {
+    initiateHelia()
+  }, [initiateHelia])
+
   return (
     <SnackbarProvider maxSnack={1}>
       <UrqlProvider value={currentSubgraphClient}>

packages/app/src/components/views/home/LandingView.tsx

@@ -12,6 +12,7 @@ import tabletHero from "../../../assets/images/tablet-hero-graphic.png"
 import paperTextureNight from "../../../assets/images/paper-texture-800-night.jpg"
 import { makeStyles } from "@mui/styles"
 import { useNavigate } from "react-router-dom"
+import { useIPFSContext } from "../../../services/ipfs/context"
 
 const benefits = [
   {
@@ -63,9 +64,19 @@ const useStyles = makeStyles(() => ({
   },
 }))
 
+const MOCK =
+  "<h1>A Revolutionary Motivational Tool for Developers</h1><p><strong>Introduction</strong></p><p>In today&#x27;s hyper-connected world, software performance is key. Yet, how do you ensure your developers are intrinsically motivated to deliver optimal performance? Introducing DevPill - the first-of-its-kind motivational tool designed to transform the way your developers approach their work.</p><p><strong>Product Overview</strong></p><p>DevPill is a groundbreaking tool that operates on a powerful yet simple premise: When your software crashes or runs slowly, your developers experience a slight discomfort. This immediate feedback mechanism provides developers a visceral connection to their work&#x27;s performance, igniting a deep-rooted motivation to deliver the best software solutions possible.</p><p><strong>Benefits</strong></p><ol type='1'><li><strong>Increased Performance:</strong> DevPill creates an immediate, tangible connection between a developers work and its outcome, resulting in increased focus on delivering high-performing, error-free software.</li><li><strong>Enhanced Motivation:</strong> Through DevPill, developers will strive for perfection, not just due to professional pride, but because their comfort is directly linked to the software&#x27;s performance.</li><li><strong>Team Accountability:</strong> DevPill fosters a culture of accountability and responsibility, ensuring every team member is dedicated to optimal software performance.</li><li><strong>Innovation &amp; Problem-Solving:</strong> With a keen desire to avoid discomfort, your developers will be more motivated than ever to troubleshoot and innovate, ensuring that problems are quickly resolved.</li></ol><p><strong>Safety &amp; Ethics</strong></p><p>DevPill is designed with a focus on safety and ethics. The discomfort triggered by software performance issues is mild and harmless, akin to a minor headache. Our product has been rigorously tested to ensure it complies with all health and safety standards.</p><p><strong>Conclusion</strong></p><p>In the competitive landscape of software development, every edge counts. DevPill provides a unique, revolutionary approach to enhancing developer motivation, accountability, and ultimately, software performance. Give your developers the tool they need to truly connect with their work. Choose DevPill.</p>"
+
 export const LandingView: React.FC = () => {
   const classes = useStyles()
   const navigate = useNavigate()
+  const { pinAction, encode } = useIPFSContext()
+  const test = async () => {
+    const textEnconded = await encode(MOCK)
+    textEnconded && pinAction(textEnconded, "test-encoded")
+
+    console.log("textEncoded", textEnconded)
+  }
   return (
     <Page>
       <Box
@@ -110,7 +121,7 @@ export const LandingView: React.FC = () => {
                 Instant web3 publications for writers, DAOs, and any Ethereum-based account.
               </Typography>
               <Box sx={{ mt: 4 }}>
-                <Button variant="contained" onClick={() => navigate("/wallet")}>
+                <Button variant="contained" onClick={() => test()}>
                   Get Started
                 </Button>
               </Box>

packages/app/src/components/views/publication/components/ArticleItem.tsx

@@ -90,6 +90,7 @@ export const ArticleItem: React.FC<ArticleItemProps> = React.memo(
     const fetchArticleContent = useCallback(async () => {
       try {
         const data = await decodeArticleContent()
+        console.log("data", data)
         if (data) {
           setArticleHtmlContent(data)
         }

packages/app/src/hooks/useIpfs.ts

@@ -3,6 +3,7 @@ import { Pinning, PinningService } from "../models/pinning"
 import axios from "axios"
 import { useNotification } from "./useNotification"
 import { getClient } from "../services/ipfs"
+import { useIPFSContext } from "../services/ipfs/context"
 
 const IPFS_GATEWAY = import.meta.env.VITE_APP_IPFS_GATEWAY
 const INFURA_IPFS_API_KEY = import.meta.env.VITE_APP_INFURA_IPFS_API_KEY
@@ -31,6 +32,7 @@ export const useIpfs = (): IpfsFunctions => {
   const [isSelectedHowToSaveArticle] = useLocalStorage<boolean | undefined>("isSelectedHowToSaveArticle", undefined)
   const [pinning] = useLocalStorage<Pinning | undefined>("pinning", undefined)
   const [ipfsNodeEndpoint] = useLocalStorage("ipfsNodeEndpoint", undefined)
+  const { decodeCID } = useIPFSContext()
   const openNotification = useNotification()
   // TODO: keeping until we find a better way to handle this
   const getClientHack = async (ipfsNodeEndpoint?: string) => {
@@ -144,17 +146,22 @@ export const useIpfs = (): IpfsFunctions => {
     return path
   }
 
+  // const getText = async (hash: string): Promise<string> => {
+  //   const client = await getClientHack(ipfsNodeEndpoint)
+  //   let str = ""
+  //   if (client) {
+  //     const res = client.cat(hash)
+  //     const decoder = new TextDecoder()
+
+  //     for await (const val of res) {
+  //       str = str + decoder.decode(val)
+  //     }
+  //   }
+  //   return str
+  // }
+  // V2 with helia
   const getText = async (hash: string): Promise<string> => {
-    const client = await getClientHack(ipfsNodeEndpoint)
-    let str = ""
-    if (client) {
-      const res = client.cat(hash)
-      const decoder = new TextDecoder()
-
-      for await (const val of res) {
-        str = str + decoder.decode(val)
-      }
-    }
+    const str = await decodeCID(hash)
     return str
   }
 

packages/app/src/index.tsx

@@ -1,5 +1,5 @@
 import React from "react"
-import ReactDOM from "react-dom"
+import { createRoot } from "react-dom/client"
 import App from "./App"
 import { HashRouter } from "react-router-dom"
 import "./index.css"
@@ -9,8 +9,12 @@ import { day } from "./theme/day"
 import { Web3ReactProvider } from "@web3-react/core"
 import { getLibrary } from "./config"
 import { Helmet } from "react-helmet"
+import { IPFSProvider } from "./services/ipfs/context"
 
-ReactDOM.render(
+const container = document.getElementById("root") as Element | DocumentFragment
+const root = createRoot(container)
+
+root.render(
   <React.StrictMode>
     <Helmet>
       <meta property="og:title" content="Tabula" />
@@ -26,11 +30,12 @@ ReactDOM.render(
     <Web3ReactProvider getLibrary={getLibrary}>
       <HashRouter>
         <ThemeProvider theme={day}>
-          <CssBaseline />
-          <App />
+          <IPFSProvider>
+            <CssBaseline />
+            <App />
+          </IPFSProvider>
         </ThemeProvider>
       </HashRouter>
     </Web3ReactProvider>
   </React.StrictMode>,
-  document.getElementById("root"),
 )

packages/app/src/services/ipfs/context/IPFS.context.tsx

@@ -0,0 +1,125 @@
+import { useState } from "react"
+import { createGenericContext } from "../../../utils/create-generic-context"
+import { IPFSContextType, IPFSProviderProps } from "./IPFS.types"
+import { Helia, createHelia } from "helia"
+import { UnixFS, unixfs } from "@helia/unixfs"
+import { LevelBlockstore } from "blockstore-level"
+import { libp2pDefaults } from "../dataTransmission/libp2p-defaults.browser"
+import { Configuration, RemotePinningServiceClient } from "@ipfs-shipyard/pinning-service-client"
+import { createRemotePinner } from "@helia/remote-pinning"
+import { multiaddr } from "@multiformats/multiaddr"
+import { bootstrapConfig } from "../dataTransmission/bootstrappers"
+
+const [useIPFSContext, IPFSContextProvider] = createGenericContext<IPFSContextType>()
+
+
+const pinServiceConfig = new Configuration({
+  endpointUrl: "http://127.0.0.1:5001/api/v0", // the URI for your pinning provider, e.g. `http://localhost:3000`
+  // accessToken:
+})
+// const pinServiceConfig = new Configuration({
+//   endpointUrl: "https://api.pinata.cloud/psa", // the URI for your pinning provider, e.g. `http://localhost:3000`
+//   accessToken:
+// })
+
+const remotePinningClient = new RemotePinningServiceClient(pinServiceConfig)
+
+const IPFSProvider = ({ children }: IPFSProviderProps) => {
+  const [helia, setHelia] = useState<Helia | undefined>(undefined)
+  const [fs, setFs] = useState<UnixFS | undefined>(undefined)
+  const [startingHelia, setStartingHelia] = useState<boolean>(false)
+
+  const startHelia = async () => {
+    if (helia) return console.log("Helia Started")
+    try {
+      setStartingHelia(true)
+      const blockstore = new LevelBlockstore(`helia-example-blockstore`)
+      console.log("Starting Helia")
+      //@ts-expect-error types are borked...
+      const heliaInstance = await createHelia({ blockstore, libp2p: libp2pDefaults() })
+      const fsInstance = unixfs(heliaInstance)
+      setHelia(heliaInstance)
+      setFs(fsInstance)
+      setStartingHelia(false)
+    } catch (error) {
+      console.error("Helia creation failed:", error)
+      setHelia(undefined)
+      setFs(undefined)
+      setStartingHelia(false)
+    }
+  }
+
+  const decodeCID = async (cid: string): Promise<string> => {
+    let text = ""
+    if (fs) {
+      // this decoder will turn Uint8Arrays into strings
+      const decoder = new TextDecoder()
+      //@ts-expect-error Error with the types
+      for await (const chunk of fs.cat(cid)) {
+        text += decoder.decode(chunk, {
+          stream: true,
+        })
+      }
+    }
+    return text
+  }
+
+  const encode = async (text: string): Promise<string | undefined> => {
+    if (!helia) return
+    if (fs && helia) {
+      const encoder = new TextEncoder()
+      try {
+        //@ts-expect-error
+        const cid = await fs.addBytes(encoder.encode(text), helia.blockstore)
+        console.log("cid", cid)
+        console.log("Added file:", cid.toString())
+        return cid as any
+      } catch (e) {
+        console.error(e)
+        return
+      }
+    }
+  }
+
+  const pinAction = async (cid: string, name: string) => {
+    if (helia) {
+      const remotePinner = createRemotePinner(helia, remotePinningClient)
+      const addPinResult = await remotePinner
+        .addPin({
+          //@ts-expect-error
+          cid,
+          name,
+          origins: new Set(bootstrapConfig.list),
+        })
+        .then((t) => console.log("t.requestedId", t.requestid))
+      const pins = await remotePinningClient.pinsGet()
+      console.log("pins", pins)
+      console.log("addPinResult", addPinResult)
+      // await getPin("9300fb6d-bae4-4b3b-a68b-8f9c43fca6de")
+    }
+  }
+  const getPin = async (requestid: string) => {
+    if (helia) {
+      const pinResults = await remotePinningClient.pinsRequestidGet({ requestid })
+      console.log("pinResults", pinResults)
+    }
+  }
+
+  return (
+    <IPFSContextProvider
+      value={{
+        helia,
+        fs,
+        startingHelia,
+        startHelia,
+        decodeCID,
+        encode,
+        pinAction,
+      }}
+    >
+      {children}
+    </IPFSContextProvider>
+  )
+}
+
+export { useIPFSContext, IPFSProvider }

packages/app/src/services/ipfs/context/IPFS.types.ts

@@ -0,0 +1,17 @@
+import { UnixFS } from "@helia/unixfs"
+import { Helia } from "helia"
+import { ReactNode } from "react"
+
+export type IPFSContextType = {
+  helia: Helia | undefined
+  fs: UnixFS | undefined
+  startingHelia: boolean
+  startHelia: () => Promise<void>
+  decodeCID: (cid: string) => Promise<string>
+  encode: (cid: string) => Promise<string | undefined>
+  pinAction: (cid: string, name: string) => Promise<void>
+}
+
+export type IPFSProviderProps = {
+  children: ReactNode
+}

packages/app/src/services/ipfs/context/index.ts

@@ -0,0 +1,2 @@
+export * from "./IPFS.context"
+export * from "./IPFS.types"

packages/app/src/services/ipfs/dataTransmission/bootstrappers.ts

@@ -0,0 +1,12 @@
+// this list comes from https://github.com/ipfs/kubo/blob/da28fbc65a2e0f1ce59f9923823326ae2bc4f713/config/bootstrap_peers.go#L17
+export const bootstrapConfig = {
+  list: [
+    "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
+    // "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
+    "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
+    "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
+    "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
+    "/ip4/104.131.131.82/udp/4001/quic/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
+    "/ip4/127.0.0.1/tcp/5001"
+  ],
+}