Dcup v. 1.2.0

Clients/RAG_Client.ts

@@ -0,0 +1,165 @@
+/* =======================
+   Response Types
+======================= */
+interface StoreResponse {
+  file_name: string,
+  file_type: string,
+  job_id: string,
+  stage: string,
+  status: string,
+}
+
+interface QueryResponse {
+  text: string
+  score: number
+  token_count: number
+  content_type: string
+  metadata: Record<string, any>
+  table_json: (string | null)[][]
+  reference: {
+    file: string
+    section: string
+    pages: number[]
+  }
+}
+
+/* =======================
+   Core Client
+======================= */
+
+class RAGClient {
+  private baseUrl: string
+
+  constructor(baseUrl: string) {
+    this.baseUrl = baseUrl
+  }
+
+  /* -----------------------
+     STORE
+  ----------------------- */
+
+  store = {
+    file: {
+      all: (file: File, metadata: any) =>
+        this.uploadFile<StoreResponse>("/store/upload/all", file, metadata),
+      pdf: (file: File, metadata?: any) =>
+        this.uploadFile<StoreResponse>("/store/upload/pdf", file, metadata),
+
+      md: (file: File, metadata?: any) =>
+        this.uploadFile("/store/upload/md", file, metadata),
+
+      csv: (file: File, metadata?: any) =>
+        this.uploadFile("/store/upload/csv", file, metadata),
+
+      json: (file: File, metadata?: any) =>
+        this.uploadFile("/store/upload/json", file, metadata),
+
+      sheet: (file: File, metadata?: any) =>
+        this.uploadFile("/store/upload/sheet", file, metadata)
+    },
+
+    url: {
+      all: (url: string, metadata?: any) =>
+        this.uploadUrl<StoreResponse>("/store/url/all", url, metadata),
+      pdf: (url: string, metadata?: any) =>
+        this.uploadUrl<StoreResponse>("/store/url/pdf", url, metadata),
+
+      md: (url: string, metadata?: any) =>
+        this.uploadUrl("/store/url/md", url, metadata),
+
+      csv: (url: string, metadata?: any) =>
+        this.uploadUrl("/store/url/csv", url, metadata),
+
+      json: (url: string, metadata?: any) =>
+        this.uploadUrl("/store/url/json", url, metadata),
+
+      sheet: (url: string, metadata?: any) =>
+        this.uploadUrl("/store/url/sheet", url, metadata)
+    }
+  }
+
+  getProgressUrl(userId: string) {
+    return `${this.baseUrl}/progress/${userId}`;
+  }
+  /* -----------------------
+     QUERY
+  ----------------------- */
+
+  async query(
+    queries: string[],
+    opts: {
+      metadata: string
+      top_result?: number | null
+    }
+  ): Promise<QueryResponse[]> {
+    const body = new URLSearchParams()
+
+    queries.forEach(q => body.append("queries", q))
+    if (opts.metadata) body.append("metadata", opts.metadata)
+    if (opts?.top_result != null)
+      body.append("top_result", String(opts.top_result))
+
+    return this.request("/query", {
+      method: "POST",
+      body
+    })
+  }
+
+  /* =======================
+     Internal helpers
+======================= */
+
+  private async uploadFile<T = any>(
+    path: string,
+    file: File,
+    metadata: any
+  ): Promise<T> {
+    const form = new FormData()
+    form.append("upload", file)
+    form.append("metadata", JSON.stringify(metadata))
+
+    return this.request(path, {
+      method: "POST",
+      body: form
+    })
+  }
+
+  private async uploadUrl<T = any>(
+    path: string,
+    url: string,
+    metadata: any
+  ): Promise<T> {
+    const body = new URLSearchParams()
+    body.append("url", url)
+    body.append("metadata", JSON.stringify(metadata))
+
+    return this.request(path, {
+      method: "POST",
+      body
+    })
+  }
+
+  private async request<T>(
+    path: string,
+    init: RequestInit
+  ): Promise<T> {
+    const res = await fetch(this.baseUrl + path, {
+      ...init,
+      headers: {
+        ...(init.body instanceof FormData
+          ? {}
+          : { "Content-Type": "application/x-www-form-urlencoded" })
+      }
+    })
+
+    if (!res.ok) {
+      const text = await res.text()
+      throw new Error(`API ${res.status}: ${text}`)
+    }
+
+    return res.json()
+  }
+}
+
+const RAG_Client = new RAGClient(process.env.DCUP_RAG ?? "http://127.0.0.1:8000")
+export default RAG_Client

DataSource/Aws/setAwsConnection.ts

@@ -1,7 +1,7 @@
 import { databaseDrizzle } from "@/db";
-import { connections } from "@/db/schemas/connections";
 import { eq } from "drizzle-orm";
 import { connectionConfig } from "../utils";
+import { connections } from "@/db/schema";
 
 
 export const setAWSConnection = async (formData: FormData) => {
@@ -16,13 +16,7 @@ export const setAWSConnection = async (formData: FormData) => {
   })
 
   if (!config.success) {
-    const errors = config.error.errors
-      .map(err => {
-        const fieldPath = err.path.length > 0 ? err.path.join('.') : 'value'
-        return `"${fieldPath}": ${err.message}`
-      })
-      .join('; ')
-    throw new Error(`Validation errors - ${errors}`)
+    throw new Error(`Validation errors - ${config.error.message}`)
   }
 
   await databaseDrizzle.update(connections).set({

DataSource/DirectUpload/SetNewConfigDirect/SetNewConfigDirect.tsx

@@ -18,7 +18,7 @@ export const SetNewConfigDirect = () => {
       <DialogTrigger asChild>
         <Button data-test={"btn-upload-files"}>Upload Files</Button>
       </DialogTrigger>
-      <DialogContent className="sm:max-w-[425px]">
+      <DialogContent className="min-w-5xl">
         <DialogHeader>
           <DialogTitle>Upload Files Directly</DialogTitle>
         </DialogHeader>

DataSource/DirectUpload/setDirectUploadConnection.ts

@@ -1,57 +1,10 @@
 import { databaseDrizzle } from "@/db";
-import { connections, processedFiles } from "@/db/schemas/connections";
+import { connections, processedFiles } from "@/db/schema";
 import { tryAndCatch } from "@/lib/try-catch";
 import { qdrant_collection_name, qdrantClient } from "@/qdrant";
 import { and, eq } from "drizzle-orm";
 import { z } from "zod";
 
-const directUploadConfig = z.object({
-  userId: z.string().min(5),
-  identifier: z.string().min(2),
-  metadata: z.string()
-    .transform((str, ctx): string => {
-      try {
-        if (str) {
-          JSON.parse(str)
-          return str
-        }
-        return "{}"
-      } catch (e) {
-        ctx.addIssue({ code: 'custom', message: 'Invalid JSON' })
-        return z.NEVER
-      }
-    }),
-  pageLimit: z.string().nullable().transform((str, ctx): number | null => {
-    try {
-      if (str) return parseInt(str)
-      return null
-    } catch (error) {
-      ctx.addIssue({ code: 'invalid_date', message: "invalid page limit" })
-      return z.NEVER
-    }
-  }),
-  fileLimit: z.string().nullable().transform((str, ctx): number | null => {
-    try {
-      if (str) return parseInt(str)
-      return null
-    } catch (error) {
-      ctx.addIssue({ code: 'invalid_date', message: "invalid page limit" })
-      return z.NEVER
-    }
-  }),
-  files: z.array(z.any().refine((file) => {
-    return (
-      file ||
-      (file instanceof File && file.type === "application/pdf")
-    );
-  },
-    {
-      message: "Invalid File",
-    })
-  ),
-  links: z.array(z.string().min(5)),
-  texts: z.array(z.string().min(5)),
-})
 
 const updateDirectUploadConfig = z.object({
   userId: z.string().min(5),
@@ -61,7 +14,7 @@ const updateDirectUploadConfig = z.object({
       if (str) return parseInt(str)
       return null
     } catch (error) {
-      ctx.addIssue({ code: 'invalid_date', message: "invalid page limit" })
+      ctx.addIssue({ code: 'custom', message: "invalid page limit" })
       return z.NEVER
     }
   }),
@@ -107,13 +60,8 @@ export const updateDirectUploadConnection = async (formData: FormData) => {
   })
 
   if (!config.success) {
-    const errors = config.error.errors
-      .map(err => {
-        const fieldPath = err.path.length > 0 ? err.path.join('.') : 'value'
-        return `"${fieldPath}": ${err.message}`
-      })
-      .join('; ')
-    throw new Error(`Validation errors - ${errors}`)
+
+    throw new Error(`Validation errors - ${config.error.message}`)
   }
 
   const connectionChunksIds: { chunksIds: string[], name: string }[] = [];
@@ -178,13 +126,7 @@ export const setDirectUploadConnection = async (formData: FormData) => {
   })
 
   if (!config.success) {
-    const errors = config.error.errors
-      .map(err => {
-        const fieldPath = err.path.length > 0 ? err.path.join('.') : 'value'
-        return `"${fieldPath}": ${err.message}`
-      })
-      .join('; ')
-    throw new Error(`Validation errors - ${errors}`)
+    throw new Error(`Validation errors - ${config.error.message}`)
   }
 
   const { files, links, userId, identifier, metadata, fileLimit, pageLimit, texts } = config.data;

DataSource/Dropbox/setDropboxConnection.ts

@@ -1,7 +1,7 @@
 import { databaseDrizzle } from "@/db";
-import { connections } from "@/db/schemas/connections";
 import { eq } from "drizzle-orm";
 import { connectionConfig } from "../utils";
+import { connections } from "@/db/schema";
 
 export const setDropboxConnection = async (formData: FormData) => {
   const config = connectionConfig.safeParse({
@@ -15,13 +15,7 @@ export const setDropboxConnection = async (formData: FormData) => {
   })
 
   if (!config.success) {
-    const errors = config.error.errors
-      .map(err => {
-        const fieldPath = err.path.length > 0 ? err.path.join('.') : 'value'
-        return `"${fieldPath}": ${err.message}`
-      })
-      .join('; ')
-    throw new Error(`Validation errors - ${errors}`)
+    throw new Error(`Validation errors - ${config.error.message}`)
   }
 
   await databaseDrizzle.update(connections).set({

DataSource/GoogleDrive/setGoogleDriveConnection.ts

@@ -1,7 +1,7 @@
 import { databaseDrizzle } from "@/db";
-import { connections } from "@/db/schemas/connections";
 import { eq } from "drizzle-orm";
 import { connectionConfig } from "../utils";
+import { connections } from "@/db/schema";
 
 export const setGoogleDriveConnection = async (formData: FormData) => {
   const config = connectionConfig.safeParse({
@@ -15,13 +15,7 @@ export const setGoogleDriveConnection = async (formData: FormData) => {
   })
 
   if (!config.success) {
-    const errors = config.error.errors
-      .map(err => {
-        const fieldPath = err.path.length > 0 ? err.path.join('.') : 'value'
-        return `"${fieldPath}": ${err.message}`
-      })
-      .join('; ')
-    throw new Error(`Validation errors - ${errors}`)
+    throw new Error(`Validation errors - ${config.error.message}`)
   }
 
   await databaseDrizzle.update(connections).set({

DataSource/utils.ts

@@ -26,7 +26,7 @@ export const connectionConfig = z.object({
       if (str) return parseInt(str)
       return null
     } catch (error) {
-      ctx.addIssue({ code: 'invalid_date', message: "invalid page limit" })
+      ctx.addIssue({ code: 'custom', message: "invalid page limit" })
       return z.NEVER
     }
   }),
@@ -35,7 +35,7 @@ export const connectionConfig = z.object({
       if (str) return parseInt(str)
       return null
     } catch (error) {
-      ctx.addIssue({ code: 'invalid_date', message: "invalid page limit" })
+      ctx.addIssue({ code: 'custom', message: "invalid page limit" })
       return z.NEVER
     }
   }),