support txt and direct text input

Author: aliamerj

.github/workflows/devs.yml

@@ -33,7 +33,8 @@ jobs:
       # 2) generate a full .env before starting anything:
       - name: Generate Env File
         run: |
-         cat <<EOF > .env 
+         cat <<EOF > .env
+         DCUP_ENV=OS
          NEXT_PUBLIC_APP_ENV=TEST
          DCUP_PARSER=http://localhost:9000
          NEXTAUTH_URL=http://localhost:3000

DataSource/Aws/setAwsConnection.ts

@@ -44,6 +44,7 @@ export const setAWSConnection = async (formData: FormData) => {
     fileLimit: config.data.fileLimit,
     metadata: config.data.metadata,
     files: [],
-    links: []
+    links: [],
+    texts: []
   }
 }

DataSource/DirectUpload/setDirectUploadConnection.ts

@@ -50,6 +50,7 @@ const directUploadConfig = z.object({
     })
   ),
   links: z.array(z.string().min(5)),
+  texts: z.array(z.string().min(5)),
 })
 
 const updateDirectUploadConfig = z.object({
@@ -88,6 +89,7 @@ const updateDirectUploadConfig = z.object({
     })
   ),
   links: z.array(z.string().min(5)),
+  texts: z.array(z.string().min(5)),
   removedFiles: z.array(z.string().min(5))
 })
 
@@ -99,6 +101,7 @@ export const updateDirectUploadConnection = async (formData: FormData) => {
     metadata: formData.get("metadata") || "{}",
     files: formData.getAll("files") || [],
     links: formData.getAll("links") || [],
+    texts: formData.getAll("texts") || [],
     removedFiles: formData.getAll("removedFiles") || [],
     pageLimit: formData.get("pageLimit")
   })
@@ -156,13 +159,13 @@ export const updateDirectUploadConnection = async (formData: FormData) => {
     metadata: config.data.metadata ?? "{}",
     files: await Promise.all(files),
     links: config.data.links,
+    texts: config.data.texts,
     pageLimit: config.data.pageLimit,
     fileLimit: null
   }
 }
 
 export const setDirectUploadConnection = async (formData: FormData) => {
-
   const config = directUploadConfig.safeParse({
     userId: formData.get("userId"),
     identifier: formData.get("identifier"),
@@ -171,6 +174,7 @@ export const setDirectUploadConnection = async (formData: FormData) => {
     fileLimit: formData.get("fileLimit"),
     files: formData.getAll("files"),
     links: formData.getAll("links"),
+    texts: formData.getAll("texts")
   })
 
   if (!config.success) {
@@ -183,9 +187,9 @@ export const setDirectUploadConnection = async (formData: FormData) => {
     throw new Error(`Validation errors - ${errors}`)
   }
 
-  const { files, links, userId, identifier, metadata, fileLimit, pageLimit } = config.data;
+  const { files, links, userId, identifier, metadata, fileLimit, pageLimit, texts } = config.data;
 
-  if (files.length === 0 && links.length === 0) {
+  if (files.length === 0 && links.length === 0 && texts.length === 0) {
     throw new Error('Please provide at least one file or link to proceed.')
   }
 
@@ -213,6 +217,7 @@ export const setDirectUploadConnection = async (formData: FormData) => {
     metadata: metadata,
     files: await Promise.all(allFiles),
     links: links,
+    texts: texts,
     pageLimit: config.data.pageLimit,
     fileLimit: fileLimit
   }

DataSource/Dropbox/setDropboxConnection.ts

@@ -43,6 +43,7 @@ export const setDropboxConnection = async (formData: FormData) => {
     fileLimit: config.data.fileLimit,
     metadata: config.data.metadata,
     files: [],
-    links: []
+    links: [],
+    texts: [],
   }
 }

DataSource/GoogleDrive/setGoogleDriveConnection.ts

@@ -43,6 +43,7 @@ export const setGoogleDriveConnection = async (formData: FormData) => {
     fileLimit: config.data.fileLimit,
     metadata: config.data.metadata,
     files: [],
-    links: []
+    links: [],
+    texts: []
   }
 }

app/api/connections/[id]/files/route.ts

@@ -109,7 +109,6 @@ export async function DELETE(request: NextRequest, { params }: Params) {
       ? [validation.data.file]
       : validation.data.files ?? [];
 
-
     for (const fileName of filesToDelete) {
       const { data, error } = await tryAndCatch(databaseDrizzle
         .delete(processedFiles)

components/Connections/Connections.tsx

@@ -47,7 +47,7 @@ export default function Connections({ connections, tokens }: { connections: Conn
     if (connProgress?.status === 'FINISHED') {
       route.refresh();
     }
-  }, [connProgress]);
+  }, [connProgress, route]);
 
   return connections.map(connection => {
     const progress = connection.id === connProgress?.connectionId ? connProgress : null;

components/UploadFileForm/UploadFileForm.tsx

@@ -32,6 +32,7 @@ type TFileForm = {
 
 export const UploadFileForm = ({ setOpen, connection }: TFileForm) => {
   const [links, setLinks] = useState<string[]>([]);
+  const [text, setText] = useState<string>("")
   const [files, setFiles] = useState<File[]>([]);
   const [removedFiles, setRemovedFiles] = useState<string[]>([]);
   const [pending, startTransition] = useTransition();
@@ -47,6 +48,7 @@ export const UploadFileForm = ({ setOpen, connection }: TFileForm) => {
           removedFiles.forEach((fileName) => data.append("removedFiles", fileName));
         } else {
           data.set("service", "DIRECT_UPLOAD");
+          if (text) data.append("texts", text)
         }
         const res = await setConnectionConfig(EMPTY_FORM_STATE, data)
         if (res.status === "SUCCESS") {
@@ -109,6 +111,7 @@ export const UploadFileForm = ({ setOpen, connection }: TFileForm) => {
         <DataInput
           files={files}
           setFiles={setFiles}
+          setText={setText}
           links={links}
           setLinks={setLinks}
           currentFiles={connection ? connection.files.map(f => f.name) : []}
@@ -139,12 +142,13 @@ type TDataInput = {
   setFiles: Dispatch<SetStateAction<File[]>>;
   links: string[];
   setLinks: Dispatch<SetStateAction<string[]>>;
+  setText: Dispatch<SetStateAction<string>>;
   currentFiles: string[];
   removedFiles: string[];
   setRemovedFiles: Dispatch<SetStateAction<string[]>>;
 };
 
-export const DataInput = ({ files, setFiles, links, setLinks, currentFiles, removedFiles, setRemovedFiles }: TDataInput) => {
+export const DataInput = ({ files, setFiles, links, setLinks, currentFiles, removedFiles, setRemovedFiles, setText }: TDataInput) => {
   const inputFile = useRef<HTMLInputElement>(null);
   const [invalidLinks, setInvalidLinks] = useState<string[]>([]);
   const [invalidFile, setInvalidFile] = useState("");
@@ -160,12 +164,12 @@ export const DataInput = ({ files, setFiles, links, setLinks, currentFiles, remo
     e.preventDefault();
     const droppedFiles = Array.from(e.dataTransfer.files);
     const validFiles = droppedFiles.filter((f) => {
-      if (f.type !== "application/pdf") {
-         setInvalidFile(`${f.name} is not supported. Please upload PDFs only.`);
+      if (f.type !== "application/pdf" && f.type !== "text/plain" && f.name.endsWith(".txt")) {
+        setInvalidFile(`${f.name} is not supported. Please upload PDFs only.`);
         return false;
       }
       if (currentFiles.includes(f.name)) {
-          setInvalidFile(`${f.name} is already added`);
+        setInvalidFile(`${f.name} is already added`);
         return false;
       }
       return true;
@@ -177,8 +181,8 @@ export const DataInput = ({ files, setFiles, links, setLinks, currentFiles, remo
     e.preventDefault();
     const selectedFiles = Array.from(e.target.files || []);
     const validFiles = selectedFiles.filter((f) => {
-      if (f.type !== "application/pdf") {
-        setInvalidFile(`${f.name} is not supported. Please upload PDFs only.`);
+      if (f.type !== "application/pdf" && f.type !== "text/plain" && f.name.endsWith(".txt")) {
+        setInvalidFile(`${f.name} is not supported. Please upload PDFs or .txt only.`);
         return false;
       }
       if (currentFiles.includes(f.name)) {
@@ -224,22 +228,25 @@ export const DataInput = ({ files, setFiles, links, setLinks, currentFiles, remo
   return (
     <div className="flex-1 flex flex-col h-full">
       <Tabs defaultValue="file" className="w-full h-full flex flex-col">
-        <TabsList className="grid w-full grid-cols-2">
+        <TabsList className="grid w-full grid-cols-3">
           <TabsTrigger value="file" className="gap-2">
             <UploadCloud className="h-5 w-5" />
-            Upload Files
+            Files
+          </TabsTrigger>
+          <TabsTrigger data-test={"btn-texts"} value="texts" className="gap-2">
+            <Link className="h-5 w-5" />
+            Texts
           </TabsTrigger>
           <TabsTrigger value="link" className="gap-2">
             <Link className="h-5 w-5" />
-            Paste Links
+            Links
           </TabsTrigger>
         </TabsList>
-
         <TabsContent value="file" className="flex-1">
           <Card className="flex flex-col">
             <CardHeader>
-              <CardTitle>Upload your PDFs</CardTitle>
-              <CardDescription>Only PDF files are supported.</CardDescription>
+              <CardTitle>Upload your Files</CardTitle>
+              <CardDescription>Only PDF,txt files are supported.</CardDescription>
               {invalidFile && (
                 <div className="mt-2 text-red-600 text-sm">
                   <AlertCircle className="h-4 w-4 inline-block mr-1" />
@@ -254,14 +261,13 @@ export const DataInput = ({ files, setFiles, links, setLinks, currentFiles, remo
                 className="border-2 border-dashed border-gray-300 rounded-xl p-5 text-center hover:border-teal-500 transition-colors flex flex-col justify-center"
               >
                 <UploadCloud className="h-12 w-12 mx-auto text-gray-400 mb-4" />
-                <p className="text-gray-600 mb-2">Drag & drop PDF files here</p>
+                <p className="text-gray-600 mb-2">Drag & drop PDF, txt files here</p>
                 <p className="text-sm text-gray-500 mb-4">or</p>
                 <input
                   type="file"
                   name="fileUpload"
                   ref={inputFile}
                   multiple
-                  accept="application/pdf"
                   onChange={addNewFile}
                   className={env === 'TEST' ? "block" : "hidden"}
                   id="file-upload"
@@ -295,8 +301,8 @@ export const DataInput = ({ files, setFiles, links, setLinks, currentFiles, remo
         <TabsContent value="link" className="flex-1">
           <Card className="flex flex-col">
             <CardHeader>
-              <CardTitle>Provide PDF URLs</CardTitle>
-              <CardDescription>Enter valid HTTP/HTTPS links to PDF files.</CardDescription>
+              <CardTitle>Provide PDF, txt URLs</CardTitle>
+              <CardDescription>Enter valid HTTP/HTTPS links to PDF, txt files.</CardDescription>
             </CardHeader>
             <CardContent>
               <div className="space-y-4">
@@ -324,6 +330,25 @@ export const DataInput = ({ files, setFiles, links, setLinks, currentFiles, remo
             </CardContent>
           </Card>
         </TabsContent>
+
+        <TabsContent value="texts" className="flex-1">
+          <Card className="flex flex-col">
+            <CardHeader>
+              <CardTitle>Provide Direcct Text</CardTitle>
+              <CardDescription>Enter Text</CardDescription>
+            </CardHeader>
+            <CardContent>
+              <div className="space-y-4">
+                <Textarea
+                  onChange={(e) => setText(e.target.value)}
+                  name="text"
+                  placeholder="Enter direct text"
+                  className={`w-full h-60 resize-none`}
+                />
+              </div>
+            </CardContent>
+          </Card>
+        </TabsContent>
       </Tabs>
     </div>
   );

cypress.config.ts

@@ -18,7 +18,7 @@ export default defineConfig({
   chromeWebSecurity: false,
   watchForFileChanges: false,
   e2e: {
-    defaultCommandTimeout: 10_000,
+    defaultCommandTimeout: 20000,
     setupNodeEvents(on) {
       on("task", {
         async getUserId({ email }: { email: string }) {