diff --git a/.github/workflows/devs.yml b/.github/workflows/devs.yml index 89082c6..8b00a75 100644 --- a/.github/workflows/devs.yml +++ b/.github/workflows/devs.yml @@ -33,7 +33,8 @@ jobs: # 2) generate a full .env before starting anything: - name: Generate Env File run: | - cat < .env + cat < .env + DCUP_ENV=OS NEXT_PUBLIC_APP_ENV=TEST DCUP_PARSER=http://localhost:9000 NEXTAUTH_URL=http://localhost:3000 diff --git a/DataSource/Aws/setAwsConnection.ts b/DataSource/Aws/setAwsConnection.ts index 1f1a292..0827fbb 100644 --- a/DataSource/Aws/setAwsConnection.ts +++ b/DataSource/Aws/setAwsConnection.ts @@ -44,6 +44,7 @@ export const setAWSConnection = async (formData: FormData) => { fileLimit: config.data.fileLimit, metadata: config.data.metadata, files: [], - links: [] + links: [], + texts: [] } } diff --git a/DataSource/DirectUpload/setDirectUploadConnection.ts b/DataSource/DirectUpload/setDirectUploadConnection.ts index 4fd6936..aec1593 100644 --- a/DataSource/DirectUpload/setDirectUploadConnection.ts +++ b/DataSource/DirectUpload/setDirectUploadConnection.ts @@ -50,6 +50,7 @@ const directUploadConfig = z.object({ }) ), links: z.array(z.string().min(5)), + texts: z.array(z.string().min(5)), }) const updateDirectUploadConfig = z.object({ @@ -88,6 +89,7 @@ const updateDirectUploadConfig = z.object({ }) ), links: z.array(z.string().min(5)), + texts: z.array(z.string().min(5)), removedFiles: z.array(z.string().min(5)) }) @@ -99,6 +101,7 @@ export const updateDirectUploadConnection = async (formData: FormData) => { metadata: formData.get("metadata") || "{}", files: formData.getAll("files") || [], links: formData.getAll("links") || [], + texts: formData.getAll("texts") || [], removedFiles: formData.getAll("removedFiles") || [], pageLimit: formData.get("pageLimit") }) @@ -156,13 +159,13 @@ export const updateDirectUploadConnection = async (formData: FormData) => { metadata: config.data.metadata ?? "{}", files: await Promise.all(files), links: config.data.links, + texts: config.data.texts, pageLimit: config.data.pageLimit, fileLimit: null } } export const setDirectUploadConnection = async (formData: FormData) => { - const config = directUploadConfig.safeParse({ userId: formData.get("userId"), identifier: formData.get("identifier"), @@ -171,6 +174,7 @@ export const setDirectUploadConnection = async (formData: FormData) => { fileLimit: formData.get("fileLimit"), files: formData.getAll("files"), links: formData.getAll("links"), + texts: formData.getAll("texts") }) if (!config.success) { @@ -183,9 +187,9 @@ export const setDirectUploadConnection = async (formData: FormData) => { throw new Error(`Validation errors - ${errors}`) } - const { files, links, userId, identifier, metadata, fileLimit, pageLimit } = config.data; + const { files, links, userId, identifier, metadata, fileLimit, pageLimit, texts } = config.data; - if (files.length === 0 && links.length === 0) { + if (files.length === 0 && links.length === 0 && texts.length === 0) { throw new Error('Please provide at least one file or link to proceed.') } @@ -213,6 +217,7 @@ export const setDirectUploadConnection = async (formData: FormData) => { metadata: metadata, files: await Promise.all(allFiles), links: links, + texts: texts, pageLimit: config.data.pageLimit, fileLimit: fileLimit } diff --git a/DataSource/Dropbox/setDropboxConnection.ts b/DataSource/Dropbox/setDropboxConnection.ts index 8810fc9..ea195a6 100644 --- a/DataSource/Dropbox/setDropboxConnection.ts +++ b/DataSource/Dropbox/setDropboxConnection.ts @@ -43,6 +43,7 @@ export const setDropboxConnection = async (formData: FormData) => { fileLimit: config.data.fileLimit, metadata: config.data.metadata, files: [], - links: [] + links: [], + texts: [], } } diff --git a/DataSource/GoogleDrive/setGoogleDriveConnection.ts b/DataSource/GoogleDrive/setGoogleDriveConnection.ts index 5f06387..77a6893 100644 --- a/DataSource/GoogleDrive/setGoogleDriveConnection.ts +++ b/DataSource/GoogleDrive/setGoogleDriveConnection.ts @@ -43,6 +43,7 @@ export const setGoogleDriveConnection = async (formData: FormData) => { fileLimit: config.data.fileLimit, metadata: config.data.metadata, files: [], - links: [] + links: [], + texts: [] } } diff --git a/app/api/connections/[id]/files/route.ts b/app/api/connections/[id]/files/route.ts index a47e3bb..604b9d1 100644 --- a/app/api/connections/[id]/files/route.ts +++ b/app/api/connections/[id]/files/route.ts @@ -109,7 +109,6 @@ export async function DELETE(request: NextRequest, { params }: Params) { ? [validation.data.file] : validation.data.files ?? []; - for (const fileName of filesToDelete) { const { data, error } = await tryAndCatch(databaseDrizzle .delete(processedFiles) diff --git a/components/Connections/Connections.tsx b/components/Connections/Connections.tsx index 063ef75..e23e67e 100644 --- a/components/Connections/Connections.tsx +++ b/components/Connections/Connections.tsx @@ -47,7 +47,7 @@ export default function Connections({ connections, tokens }: { connections: Conn if (connProgress?.status === 'FINISHED') { route.refresh(); } - }, [connProgress]); + }, [connProgress, route]); return connections.map(connection => { const progress = connection.id === connProgress?.connectionId ? connProgress : null; diff --git a/components/UploadFileForm/UploadFileForm.tsx b/components/UploadFileForm/UploadFileForm.tsx index 73c0254..b57ce51 100644 --- a/components/UploadFileForm/UploadFileForm.tsx +++ b/components/UploadFileForm/UploadFileForm.tsx @@ -32,6 +32,7 @@ type TFileForm = { export const UploadFileForm = ({ setOpen, connection }: TFileForm) => { const [links, setLinks] = useState([]); + const [text, setText] = useState("") const [files, setFiles] = useState([]); const [removedFiles, setRemovedFiles] = useState([]); const [pending, startTransition] = useTransition(); @@ -47,6 +48,7 @@ export const UploadFileForm = ({ setOpen, connection }: TFileForm) => { removedFiles.forEach((fileName) => data.append("removedFiles", fileName)); } else { data.set("service", "DIRECT_UPLOAD"); + if (text) data.append("texts", text) } const res = await setConnectionConfig(EMPTY_FORM_STATE, data) if (res.status === "SUCCESS") { @@ -109,6 +111,7 @@ export const UploadFileForm = ({ setOpen, connection }: TFileForm) => { f.name) : []} @@ -139,12 +142,13 @@ type TDataInput = { setFiles: Dispatch>; links: string[]; setLinks: Dispatch>; + setText: Dispatch>; currentFiles: string[]; removedFiles: string[]; setRemovedFiles: Dispatch>; }; -export const DataInput = ({ files, setFiles, links, setLinks, currentFiles, removedFiles, setRemovedFiles }: TDataInput) => { +export const DataInput = ({ files, setFiles, links, setLinks, currentFiles, removedFiles, setRemovedFiles, setText }: TDataInput) => { const inputFile = useRef(null); const [invalidLinks, setInvalidLinks] = useState([]); const [invalidFile, setInvalidFile] = useState(""); @@ -160,12 +164,12 @@ export const DataInput = ({ files, setFiles, links, setLinks, currentFiles, remo e.preventDefault(); const droppedFiles = Array.from(e.dataTransfer.files); const validFiles = droppedFiles.filter((f) => { - if (f.type !== "application/pdf") { - setInvalidFile(`${f.name} is not supported. Please upload PDFs only.`); + if (f.type !== "application/pdf" && f.type !== "text/plain" && f.name.endsWith(".txt")) { + setInvalidFile(`${f.name} is not supported. Please upload PDFs only.`); return false; } if (currentFiles.includes(f.name)) { - setInvalidFile(`${f.name} is already added`); + setInvalidFile(`${f.name} is already added`); return false; } return true; @@ -177,8 +181,8 @@ export const DataInput = ({ files, setFiles, links, setLinks, currentFiles, remo e.preventDefault(); const selectedFiles = Array.from(e.target.files || []); const validFiles = selectedFiles.filter((f) => { - if (f.type !== "application/pdf") { - setInvalidFile(`${f.name} is not supported. Please upload PDFs only.`); + if (f.type !== "application/pdf" && f.type !== "text/plain" && f.name.endsWith(".txt")) { + setInvalidFile(`${f.name} is not supported. Please upload PDFs or .txt only.`); return false; } if (currentFiles.includes(f.name)) { @@ -224,22 +228,25 @@ export const DataInput = ({ files, setFiles, links, setLinks, currentFiles, remo return (
- + - Upload Files + Files + + + + Texts - Paste Links + Links - - Upload your PDFs - Only PDF files are supported. + Upload your Files + Only PDF,txt files are supported. {invalidFile && (
@@ -254,14 +261,13 @@ export const DataInput = ({ files, setFiles, links, setLinks, currentFiles, remo className="border-2 border-dashed border-gray-300 rounded-xl p-5 text-center hover:border-teal-500 transition-colors flex flex-col justify-center" > -

Drag & drop PDF files here

+

Drag & drop PDF, txt files here

or

- Provide PDF URLs - Enter valid HTTP/HTTPS links to PDF files. + Provide PDF, txt URLs + Enter valid HTTP/HTTPS links to PDF, txt files.
@@ -324,6 +330,25 @@ export const DataInput = ({ files, setFiles, links, setLinks, currentFiles, remo + + + + + Provide Direcct Text + Enter Text + + +
+