Support for Freeforms Workflow Type Generate & Re-generate

.gitignore

@@ -57,6 +57,7 @@ Khauneesh/
 # DB
 *metadata.db-shm
 *metadata.db-wal
+telemetry.db
 
 # Test and coverage reports
 .coverage

.project-metadata.yaml

@@ -30,6 +30,10 @@ environment_variables:
     default: "your huggingface username"
     description: >-
       hf_username
+  CDP_TOKEN:
+    default: "API key for Cloudera AI Inference"
+    description: >-
+      CDP_TOKEN
 
 
 
@@ -69,7 +73,7 @@ tasks:
     script: build/build_client.py
     arguments: None
     cpu: 2
-    memory: 2
+    memory: 4
     short_summary: Create job to build client application
     environment:
       TASK_TYPE: CREATE/RUN_JOB

alembic/versions/1a8fdc23eb6f_add_s3_export_path.py

@@ -0,0 +1,38 @@
+"""add_s3_export_path
+
+Revision ID: 1a8fdc23eb6f
+Revises: 9023b46c8d4c
+Create Date: 2025-04-22 20:01:13.247491
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '1a8fdc23eb6f'
+down_revision: Union[str, None] = '9023b46c8d4c'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Add s3_export_path column to generation_metadata table
+    with op.batch_alter_table('generation_metadata', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('s3_export_path', sa.Text(), nullable=True))
+
+    # Add s3_export_path column to export_metadata table
+    with op.batch_alter_table('export_metadata', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('s3_export_path', sa.Text(), nullable=True))
+
+
+def downgrade() -> None:
+    # Remove s3_export_path column from generation_metadata table
+    with op.batch_alter_table('generation_metadata', schema=None) as batch_op:
+        batch_op.drop_column('s3_export_path')
+
+    # Remove s3_export_path column from export_metadata table
+    with op.batch_alter_table('export_metadata', schema=None) as batch_op:
+        batch_op.drop_column('s3_export_path')

app/client/eslint.config.js

@@ -23,6 +23,7 @@ export default tseslint.config(
         'warn',
         { allowConstantExport: true },
       ],
+      '@typescript-eslint/no-explicit-any': ['warn', { 'fixToUnknown': true, 'ignoreRestArgs': false }]
     },
   },
 )

app/client/package.json

@@ -16,6 +16,8 @@
     "@mui/icons-material": "6.1.7",
     "@mui/material": "6.1.7",
     "@tanstack/react-query": "5.66.0",
+    "ag-grid-community": "33.2.4",
+    "ag-grid-react":"33.2.4",
     "antd": "5.22.1",
     "axios": "1.6.7",
     "lodash": "4.17.21",

app/client/src/Container.tsx

@@ -48,9 +48,6 @@ const PageHeader = styled(Header)`
   height: fit-content;
   padding: 5px 15px
 `;
-const StyledImg = styled.img`
-  height: ${props => props?.height && `${props.height}px`}
-`
 
 const StyledText = styled.div`
   font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, 'Noto Sans', sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol', 'Noto Color Emoji';

app/client/src/api/api.ts

@@ -27,8 +27,11 @@ export const useFetchModels = (): UseFetchApiReturn<FetchModelsResp> => {
     return useFetch(url);
 }
 
-export const useFetchDefaultPrompt = (useCase: string): UseFetchApiReturn<FetchDefaultPromptResp> => {
-    const url = `${baseUrl}/${isEmpty(useCase) ? 'custom' : useCase}/gen_prompt`;
+export const useFetchDefaultPrompt = (useCase: string, workflowType?: WorkerType): UseFetchApiReturn<FetchDefaultPromptResp> => {
+    let url = `${baseUrl}/${isEmpty(useCase) ? 'custom' : useCase}/gen_prompt`;
+    if (workflowType && workflowType === 'freeform') {
+        url = `${baseUrl}/${isEmpty(useCase) ? 'custom' : useCase}/gen_freeform_prompt`;
+    }
     return useFetch(url);
 }
 
@@ -42,7 +45,7 @@ export const useFetchDefaultModelParams = (): UseFetchApiReturn<FetchDefaultPara
     return useFetch(url);
 }
 
-export const useTriggerDatagen = <T>() => {
-    const genDatasetUrl = `${import.meta.env.VITE_AMP_URL}/synthesis/generate`;
+export const useTriggerDatagen = <T>(workflow_type: string) => {
+    const genDatasetUrl = `${import.meta.env.VITE_AMP_URL}/synthesis/${workflow_type === 'freeform' ? 'freeform' : 'generate'}`;
     return usePostApi<T>(genDatasetUrl);
 }

app/client/src/api/hooks.ts

@@ -106,15 +106,15 @@ interface UsePostApiReturn<T> {
   data: T | null;
   loading: boolean;
   error: Error | null;
-  triggerPost: (body: Record<string, any>) => Promise<void>;
+  triggerPost: (body: Record<string, unknown>) => Promise<void>;
 }
 
 export function usePostApi<T>(url: string): UsePostApiReturn<T> {
   const [data, setData] = useState<T | null>(null);
   const [loading, setLoading] = useState(false);
   const [error, setError] = useState<Error | null>(null);
 
-  const triggerPost = async (body: Record<string, any>) => {
+  const triggerPost = async (body: Record<string, unknown>) => {
     setLoading(true);
     setError(null); // Reset error on each request
 

app/client/src/components/RouteAccessControl.tsx

@@ -10,7 +10,7 @@ import { Navigate, useLocation } from "react-router-dom";
  */
 interface RouteACProps{
   element: ReactNode;
-  validator: (state: any | null) => boolean;
+  validator: (state: unknown | null) => boolean;
   reroutePath?: string;
 }
 const RouteAccessControl: FC<RouteACProps> = ({ element, validator, reroutePath = '/' }) => {

app/client/src/components/TelemetryDashboard.tsx

@@ -1,15 +1,15 @@
 import React, { useState, useEffect } from 'react';
 import {
   BarChart, Bar, XAxis, YAxis, CartesianGrid, Tooltip, Legend, ResponsiveContainer,
-  LineChart, Line, AreaChart, Area
+  LineChart, Line
 } from 'recharts';
 import axios from 'axios';
 import {
   Card, Typography, Row, Col, Statistic, Select, Spin, Empty, Table, Tag, Tabs, Alert, Progress, Space, Badge, Button
 } from 'antd';
 import {
   DashboardOutlined, ApiOutlined, CloudServerOutlined, RocketOutlined, SyncOutlined,
-  PieChartOutlined, BarChartOutlined, CodeOutlined, WarningOutlined, CheckCircleOutlined, CloseCircleOutlined
+  CodeOutlined, WarningOutlined, CheckCircleOutlined, CloseCircleOutlined
 } from '@ant-design/icons';
 
 const { Title, Text } = Typography;
@@ -19,7 +19,7 @@ const SUCCESS_COLOR = '#52c41a';
 const ERROR_COLOR = '#f5222d';
 const WARNING_COLOR = '#faad14';
 const INFO_COLOR = '#1890ff';
-const COLORS = ['#0088FE', '#00C49F', '#FFBB28', '#FF8042', '#8884d8', '#4CAF50', '#F44336', '#9C27B0'];
+// const COLORS = ['#0088FE', '#00C49F', '#FFBB28', '#FF8042', '#8884d8', '#4CAF50', '#F44336', '#9C27B0'];
 
 const TelemetryDashboard = () => {
   const [loading, setLoading] = useState(true);

app/client/src/pages/DataGenerator/Configure.tsx

@@ -1,5 +1,6 @@
 import endsWith from 'lodash/endsWith';
 import isEmpty from 'lodash/isEmpty';
+import isFunction from 'lodash/isFunction';
 import { useEffect, useState } from 'react';
 import { Flex, Form, Input, Select, Typography } from 'antd';
 import styled from 'styled-components';
@@ -10,6 +11,7 @@ import { ModelProviders, ModelProvidersDropdownOpts } from './types';
 import { useWizardCtx } from './utils';
 import FileSelectorButton from './FileSelectorButton';
 
+
 const StepContainer = styled(Flex)`
     background: white;
     padding: 40px 0px;
@@ -31,7 +33,8 @@ export const USECASE_OPTIONS = [
 
 export const WORKFLOW_OPTIONS = [
     { label: 'Supervised Fine-Tuning', value: 'supervised-fine-tuning' },
-    { label: 'Custom Data Generation', value: 'custom' }
+    { label: 'Custom Data Generation', value: 'custom' },
+    { label: 'Freefrom Data Generation', value: 'freeform' }
 ];
 
 export const MODEL_TYPE_OPTIONS: ModelProvidersDropdownOpts = [
@@ -55,16 +58,23 @@ const Configure = () => {
         delete values.output_value;
 
         const allFieldsFilled = Object.values(values).every(value => Boolean(value));
-        if (allFieldsFilled) {
-            setIsStepValid && setIsStepValid(true)
-        } else {
-            setIsStepValid && setIsStepValid(false)
+        if (allFieldsFilled && isFunction(setIsStepValid)) {
+            setIsStepValid(true)
+        } else if (isFunction(setIsStepValid)) {
+            setIsStepValid(false)
         }
     }
     useEffect(() => {
         validateForm()
     }, [form, formData])
 
+    // keivan
+    useEffect(() => {
+        if (formData && formData?.inference_type === undefined) {
+            form.setFieldValue('inference_type', ModelProviders.CAII);
+        }
+    }, [formData]);
+
     const labelCol = {
       span: 8
     };
@@ -83,7 +93,7 @@ const Configure = () => {
         form.setFieldValue('doc_paths', paths);
     }
 
-    const onFilesChange = (selections: any) => {
+    const onFilesChange = (selections: unknown) => {
         if (Array.isArray(selections) && !isEmpty(selections)) {
             const paths = selections.map((file: File) => (
                 { 
@@ -106,7 +116,6 @@ const Configure = () => {
             setSelectedFiles([]);    
         }
     }
-
 
     return (
         <StepContainer justify='center'>
@@ -209,7 +218,8 @@ const Configure = () => {
                         )}
                     </Select>
                 </Form.Item>
-                {formData?.workflow_type === WorkflowType.SUPERVISED_FINE_TUNING && 
+                {(formData?.workflow_type === WorkflowType.SUPERVISED_FINE_TUNING || 
+                 formData?.workflow_type === WorkflowType.FREE_FORM_DATA_GENERATION) && 
                 <Form.Item
                     name='use_case'
                     label='Template'
@@ -234,7 +244,7 @@ const Configure = () => {
                     formData?.workflow_type === WorkflowType.CUSTOM_DATA_GENERATION) && 
                 <Form.Item
                     name='doc_paths'
-                    label='Files'
+                    label='Context'
                     labelCol={labelCol}
                     dependencies={['workflow_type']}
                     shouldUpdate
@@ -319,6 +329,23 @@ const Configure = () => {
                         <Input />
                     </Form.Item>
                 </>}
+                {/* {formData?.workflow_type === WorkflowType.FREE_FORM_DATA_GENERATION || 
+                 <Form.Item
+                    name='example_path'
+                    label='Example File'
+                    labelCol={labelCol}
+                    dependencies={['workflow_type']}
+                    shouldUpdate
+                    validateTrigger="['onBlur','onChange']"
+                    validateFirst
+                    rules={[]}
+                 >
+                    <Flex>
+                        <Select placeholder={'Select example file'} value={selectedFiles || []} onChange={onFilesChange} allowClear/>
+                        <Input placeholder='Select example file' disabled />
+                        <FileSelectorButton onAddFiles={onAddExampleFiles} workflowType={form.getFieldValue('workflow_type')} />
+                    </Flex>
+                 </Form.Item>} */}
             </FormContainer>
         </StepContainer>
     )

app/client/src/pages/DataGenerator/CustomPromptButton.tsx

@@ -1,9 +1,8 @@
-import { Button, Flex, Form, Input, Modal, notification, Spin } from "antd";
+import { Button, Form, Input, Modal, notification } from "antd";
 import { useEffect, useState } from "react";
 import { useMutation } from "@tanstack/react-query";
 import styled from "styled-components";
-import { LoadingOutlined } from '@ant-design/icons';
-import { fetchCustomPrompt, fetchPrompt } from "./hooks";
+import { fetchCustomPrompt } from "./hooks";
 import Loading from "../Evaluator/Loading";
 
 interface Props {
@@ -16,9 +15,30 @@ interface Props {
 
 export const StyledTextArea = styled(Input.TextArea)`
     margin-bottom: 10px !important;
-    min-height: 175px !important;
+    min-height: 275px !important;
+    margin-bottom: 10px !important;
+    padding: 15px 20px !important;
 `;
 
+const StyledModal = styled(Modal)`
+  .ant-modal-content {
+    max-height: 90vh;
+    // height: 760px;
+    height: 85vh;
+    width: 750px;
+    .ant-modal-body {
+      padding-top: 0;
+      min-height: 70vh;
+    }
+  }
+  // .ant-modal-content {
+  //       border-radius: 8px;
+  //       box-shadow: 0px 4px 16px rgba(0, 0, 0, 0.1);
+  //       background-color: #ffffff;
+  //       padding: 24px;
+  //  }
+`        
+
 const CustomPromptButton: React.FC<Props> = ({ model_id, inference_type, caii_endpoint, use_case, setPrompt }) => {
   const [form] = Form.useForm();
   const [showModal, setShowModal] = useState(false);
@@ -39,7 +59,7 @@ const CustomPromptButton: React.FC<Props> = ({ model_id, inference_type, caii_en
         setShowModal(false);
       }
   }, [mutation.error, mutation.isSuccess]);
-
+  
   const onFinish = async () => {
     const custom_prompt = form.getFieldValue('custom_prompt_instructions');
     try { 
@@ -67,7 +87,7 @@ const CustomPromptButton: React.FC<Props> = ({ model_id, inference_type, caii_en
       <Button onClick={() => setShowModal(true)} style={{ marginLeft: '8px' }}>Generate Custom Prompt</Button>
       {showModal && 
         (
-            <Modal
+            <StyledModal
               visible={showModal}
               okText={`Generate`}
               title={`Generate Cutom Prompt`}
@@ -80,16 +100,18 @@ const CustomPromptButton: React.FC<Props> = ({ model_id, inference_type, caii_en
                     initialValues={initialValues} 
                     onFinish={onSubmit}
                     style={{ marginTop: '24px' }}
-                    disabled={mutation.isLoading}
+                    disabled={mutation.isPending}
                 >
-                    {mutation.isLoading && 
+                    {mutation.isPending && 
                       <Loading />
                     }
 
                     <Form.Item
                         name='custom_prompt_instructions'
                         label='Custom Prompt Instructions'
                         rules={[{ required: true, message: "This field is required." }]}
+                        labelCol={{ span: 24 }}
+                        wrapperCol={{ span: 24 }}
                     >
                         <StyledTextArea 
                             autoSize 
@@ -98,7 +120,7 @@ const CustomPromptButton: React.FC<Props> = ({ model_id, inference_type, caii_en
                     </Form.Item>
                 </Form>
 
-            </Modal>
+            </StyledModal>
         )
       }
     </>