Skip to content

Checkin for free form data generation #65

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Checkin for free form data generation #65

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .project-metadata.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ tasks:
script: build/build_client.py
arguments: None
cpu: 2
memory: 2
memory: 4
short_summary: Create job to build client application
environment:
TASK_TYPE: CREATE/RUN_JOB
Expand Down
1 change: 1 addition & 0 deletions app/client/eslint.config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ export default tseslint.config(
'warn',
{ allowConstantExport: true },
],
'@typescript-eslint/no-explicit-any': ['warn', { 'fixToUnknown': true, 'ignoreRestArgs': false }]
},
},
)
2 changes: 2 additions & 0 deletions app/client/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
"@mui/icons-material": "6.1.7",
"@mui/material": "6.1.7",
"@tanstack/react-query": "5.66.0",
"ag-grid-community": "33.2.4",
"ag-grid-react":"33.2.4",
"antd": "5.22.1",
"axios": "1.6.7",
"lodash": "4.17.21",
Expand Down
11 changes: 7 additions & 4 deletions app/client/src/api/api.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,11 @@ export const useFetchModels = (): UseFetchApiReturn<FetchModelsResp> => {
return useFetch(url);
}

export const useFetchDefaultPrompt = (useCase: string): UseFetchApiReturn<FetchDefaultPromptResp> => {
const url = `${baseUrl}/${isEmpty(useCase) ? 'custom' : useCase}/gen_prompt`;
export const useFetchDefaultPrompt = (useCase: string, workflowType?: WorkerType): UseFetchApiReturn<FetchDefaultPromptResp> => {
let url = `${baseUrl}/${isEmpty(useCase) ? 'custom' : useCase}/gen_prompt`;
if (workflowType && workflowType === 'freeform') {
url = `${baseUrl}/${isEmpty(useCase) ? 'custom' : useCase}/gen_freeform_prompt`;
}
return useFetch(url);
}

Expand All @@ -42,7 +45,7 @@ export const useFetchDefaultModelParams = (): UseFetchApiReturn<FetchDefaultPara
return useFetch(url);
}

export const useTriggerDatagen = <T>() => {
const genDatasetUrl = `${import.meta.env.VITE_AMP_URL}/synthesis/generate`;
export const useTriggerDatagen = <T>(workflow_type: string) => {
const genDatasetUrl = `${import.meta.env.VITE_AMP_URL}/synthesis/${workflow_type === 'freeform' ? 'freeform' : 'generate'}`;
return usePostApi<T>(genDatasetUrl);
}
43 changes: 40 additions & 3 deletions app/client/src/pages/DataGenerator/Configure.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import { MODEL_PROVIDER_LABELS } from './constants';
import { ModelProviders, ModelProvidersDropdownOpts } from './types';
import { useWizardCtx } from './utils';
import FileSelectorButton from './FileSelectorButton';
import first from 'lodash/first';
import get from 'lodash/get';

const StepContainer = styled(Flex)`
background: white;
Expand All @@ -31,7 +33,8 @@ export const USECASE_OPTIONS = [

export const WORKFLOW_OPTIONS = [
{ label: 'Supervised Fine-Tuning', value: 'supervised-fine-tuning' },
{ label: 'Custom Data Generation', value: 'custom' }
{ label: 'Custom Data Generation', value: 'custom' },
{ label: 'Freefrom Data Generation', value: 'freeform' }
];

export const MODEL_TYPE_OPTIONS: ModelProvidersDropdownOpts = [
Expand Down Expand Up @@ -65,6 +68,13 @@ const Configure = () => {
validateForm()
}, [form, formData])

// keivan
useEffect(() => {
if (formData && formData?.inference_type === undefined) {
form.setFieldValue('inference_type', ModelProviders.CAII);
}
}, [formData]);

const labelCol = {
span: 8
};
Expand Down Expand Up @@ -106,7 +116,16 @@ const Configure = () => {
setSelectedFiles([]);
}
}

const onAddExampleFiles = (files: File[]) => {
console.log('onAddExampleFiles', files);
if (!isEmpty(files)) {
const file = first(files);
form.setFieldValue('example_path', get(file, '_path'));
}
}

console.log('formData', formData);

return (
<StepContainer justify='center'>
Expand Down Expand Up @@ -209,7 +228,8 @@ const Configure = () => {
)}
</Select>
</Form.Item>
{formData?.workflow_type === WorkflowType.SUPERVISED_FINE_TUNING &&
{(formData?.workflow_type === WorkflowType.SUPERVISED_FINE_TUNING ||
formData?.workflow_type === WorkflowType.FREE_FORM_DATA_GENERATION) &&
<Form.Item
name='use_case'
label='Template'
Expand All @@ -234,7 +254,7 @@ const Configure = () => {
formData?.workflow_type === WorkflowType.CUSTOM_DATA_GENERATION) &&
<Form.Item
name='doc_paths'
label='Files'
label='Context'
labelCol={labelCol}
dependencies={['workflow_type']}
shouldUpdate
Expand Down Expand Up @@ -319,6 +339,23 @@ const Configure = () => {
<Input />
</Form.Item>
</>}
{/* {formData?.workflow_type === WorkflowType.FREE_FORM_DATA_GENERATION ||
<Form.Item
name='example_path'
label='Example File'
labelCol={labelCol}
dependencies={['workflow_type']}
shouldUpdate
validateTrigger="['onBlur','onChange']"
validateFirst
rules={[]}
>
<Flex>
<Select placeholder={'Select example file'} value={selectedFiles || []} onChange={onFilesChange} allowClear/>
<Input placeholder='Select example file' disabled />
<FileSelectorButton onAddFiles={onAddExampleFiles} workflowType={form.getFieldValue('workflow_type')} />
</Flex>
</Form.Item>} */}
</FormContainer>
</StepContainer>
)
Expand Down
24 changes: 22 additions & 2 deletions app/client/src/pages/DataGenerator/CustomPromptButton.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,25 @@ export const StyledTextArea = styled(Input.TextArea)`
min-height: 175px !important;
`;

const StyledModal = styled(Modal)`
.ant-modal-content {
max-height: 90vh;
// height: 760px;
height: 85vh;
width: 750px;
.ant-modal-body {
padding-top: 0;
min-height: 70vh;
}
}
// .ant-modal-content {
// border-radius: 8px;
// box-shadow: 0px 4px 16px rgba(0, 0, 0, 0.1);
// background-color: #ffffff;
// padding: 24px;
// }
`

const CustomPromptButton: React.FC<Props> = ({ model_id, inference_type, caii_endpoint, use_case, setPrompt }) => {
const [form] = Form.useForm();
const [showModal, setShowModal] = useState(false);
Expand All @@ -40,6 +59,7 @@ const CustomPromptButton: React.FC<Props> = ({ model_id, inference_type, caii_en
}
}, [mutation.error, mutation.isSuccess]);

console.log('mutation', mutation);
const onFinish = async () => {
const custom_prompt = form.getFieldValue('custom_prompt_instructions');
try {
Expand Down Expand Up @@ -67,7 +87,7 @@ const CustomPromptButton: React.FC<Props> = ({ model_id, inference_type, caii_en
<Button onClick={() => setShowModal(true)} style={{ marginLeft: '8px' }}>Generate Custom Prompt</Button>
{showModal &&
(
<Modal
<StyledModal
visible={showModal}
okText={`Generate`}
title={`Generate Cutom Prompt`}
Expand Down Expand Up @@ -98,7 +118,7 @@ const CustomPromptButton: React.FC<Props> = ({ model_id, inference_type, caii_en
</Form.Item>
</Form>

</Modal>
</StyledModal>
)
}
</>
Expand Down
15 changes: 13 additions & 2 deletions app/client/src/pages/DataGenerator/DataGenerator.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import Finish from './Finish';

import { DataGenWizardSteps, WizardStepConfig, WorkflowType } from './types';
import { WizardCtx } from './utils';
import { useGetDatasetDetails } from '../DatasetDetails/hooks';

const { Content } = Layout;
// const { Title } = Typography;
Expand Down Expand Up @@ -98,10 +99,14 @@ const DataGenerator = () => {
const [isStepValid, setIsStepValid] = useState<boolean>(false);
// Data passed from listing table to prepopulate form
const location = useLocation();
console.log('DatGenerator >> location?.state?.data:', location?.state?.data);
console.log('location?.state?.data:', location?.state?.data);
const initialData = location?.state?.data;

const datasetDetailsReq = location?.state?.data && useGetDatasetDetails(location?.state?.data?.generate_file_name)
if (initialData?.technique) {
initialData.workflow_type = initialData?.technique === 'sft' ? WorkflowType.SUPERVISED_FINE_TUNING :
initialData.workflow_type = initialData?.technique === 'sft' ?
WorkflowType.SUPERVISED_FINE_TUNING :
initialData?.technique === 'freeform' ? WorkflowType.FREE_FORM_DATA_GENERATION :
WorkflowType.CUSTOM_DATA_GENERATION;
}
if (Array.isArray(initialData?.doc_paths) && !isEmpty(initialData?.doc_paths) ) {
Expand All @@ -111,6 +116,12 @@ const DataGenerator = () => {
}));

}

if (datasetDetailsReq && datasetDetailsReq.data &&
!isEmpty(datasetDetailsReq?.data?.generate_file_name)) {
initialData.example_path = initialData?.example_path;
}

if (Array.isArray(initialData?.input_paths) && !isEmpty(initialData?.input_paths) ) {
initialData.doc_paths = initialData?.input_paths.map((path: string) => ({
value: path,
Expand Down
Loading