'hover-pointer'}
+ rowKey={(_record, index) => `topic-generation-table-${index}`}
+ pagination={{
+ showSizeChanger: true,
+ showQuickJumper: false,
+ hideOnSinglePage: true
+ }}
+ />
+
+
+ )
+}
+
+export default TopicGenerationTable;
\ No newline at end of file
diff --git a/app/client/src/pages/DatasetDetails/constants.tsx b/app/client/src/pages/DatasetDetails/constants.tsx
new file mode 100644
index 0000000..4faa3b4
--- /dev/null
+++ b/app/client/src/pages/DatasetDetails/constants.tsx
@@ -0,0 +1,27 @@
+import { HomeOutlined, PageviewOutlined } from '@mui/icons-material';
+import AssessmentIcon from '@mui/icons-material/Assessment';
+import CheckCircleIcon from '@mui/icons-material/CheckCircle'
+import GradingIcon from '@mui/icons-material/Grading';
+import ModelTrainingIcon from '@mui/icons-material/ModelTraining';
+
+
+export const nextStepsList = [
+ {
+ avatar: '',
+ title: 'Review Dataset',
+ description: 'Review your dataset to ensure it properly fits your usecase.',
+ icon:
+ },
+ {
+ avatar: '',
+ title: 'Evaluate Dataset',
+ description: 'Use an LLM as a judge to evaluate and score your dataset.',
+ icon: ,
+ },
+ {
+ avatar: '',
+ title: 'Fine Tuning Studio',
+ description: 'Bring your dataset to Fine Tuning Studio AMP to start fine tuning your models in Cloudera AI Workbench.',
+ icon: ,
+ },
+]
\ No newline at end of file
diff --git a/app/client/src/pages/DatasetDetails/hooks.tsx b/app/client/src/pages/DatasetDetails/hooks.tsx
index e69de29..a403ae8 100644
--- a/app/client/src/pages/DatasetDetails/hooks.tsx
+++ b/app/client/src/pages/DatasetDetails/hooks.tsx
@@ -0,0 +1,63 @@
+import get from 'lodash/get';
+import { notification } from 'antd';
+import { useQuery } from 'react-query';
+
+
+const BASE_API_URL = import.meta.env.VITE_AMP_URL;
+
+
+const {
+ VITE_WORKBENCH_URL,
+ VITE_PROJECT_OWNER,
+ VITE_CDSW_PROJECT
+} = import.meta.env
+
+
+
+const fetchDatasetDetails = async (generate_file_name: string) => {
+ const dataset_details__resp = await fetch(`${BASE_API_URL}/dataset_details/${generate_file_name}`, {
+ method: 'GET',
+ });
+ const datasetDetails = await dataset_details__resp.json();
+ const dataset__resp = await fetch(`${BASE_API_URL}/generations/${generate_file_name}`, {
+ method: 'GET',
+ });
+ const dataset = await dataset__resp.json();
+
+ return {
+ dataset,
+ datasetDetails
+ };
+};
+
+
+
+
+export const useGetDatasetDetails = (generate_file_name: string) => {
+ const { data, isLoading, isError, error } = useQuery(
+ ["data", fetchDatasetDetails],
+ () => fetchDatasetDetails(generate_file_name),
+ {
+ keepPreviousData: true,
+ },
+ );
+
+ const dataset = get(data, 'dataset');
+ console.log('data:', data);
+ console.log('error:', error);
+
+ if (error) {
+ notification.error({
+ message: 'Error',
+ description: `An error occurred while fetching the dataset details:\n ${error}`
+ });
+ }
+
+ return {
+ data,
+ dataset,
+ isLoading,
+ isError,
+ error
+ };
+}
\ No newline at end of file
diff --git a/app/client/src/pages/Home/DatasetActions.tsx b/app/client/src/pages/Home/DatasetActions.tsx
index 30320ed..28e373e 100644
--- a/app/client/src/pages/Home/DatasetActions.tsx
+++ b/app/client/src/pages/Home/DatasetActions.tsx
@@ -76,12 +76,16 @@ const DatasetActions: React.FC = ({ dataset, refetch, setTo
const menuActions: MenuProps['items'] = [
{
key: '1',
- label: (
-
+ // label: (
+ //
+ // View Dataset Details
+ //
+ // ),
+ label:
+
View Dataset Details
-
- ),
- onClick: () => setShowModal(true),
+ ,
+ // onClick: () => setShowModal(true),
icon:
},
{
diff --git a/app/client/src/pages/Home/UpgradeButton.tsx b/app/client/src/pages/Home/UpgradeButton.tsx
new file mode 100644
index 0000000..b00121b
--- /dev/null
+++ b/app/client/src/pages/Home/UpgradeButton.tsx
@@ -0,0 +1,35 @@
+import isEmpty from 'lodash/isEmpty';
+import { Button } from 'antd';
+import React, { useEffect } from 'react';
+import { useUpgradeStatus } from './hooks';
+
+
+
+
+const UpgradeButton: React.FC = () => {
+ const [showModal, setShowModal] = React.useState(false);
+ const [enableUpgrade, setEnableUpgrade] = React.useState(false);
+ const { data, isLoading, isError } = useUpgradeStatus();
+
+ useEffect(() => {
+ if (isEmpty(data)) {
+ setEnableUpgrade(data?.updates_available);
+ }
+ },[data, isLoading, isError]);
+
+ const onUpgrade = () => {
+ // Logic to handle upgrade
+ console.log("Upgrading...");
+ }
+
+ if (!enableUpgrade) {
+ return null;
+ }
+
+ return (
+
+ )
+
+}
+
+export default UpgradeButton;
diff --git a/app/client/src/pages/Home/hooks.ts b/app/client/src/pages/Home/hooks.ts
index 15860a5..3d416d3 100644
--- a/app/client/src/pages/Home/hooks.ts
+++ b/app/client/src/pages/Home/hooks.ts
@@ -93,4 +93,30 @@ export const useEvaluations = () => {
searchQuery,
setSearchQuery
};
+}
+
+const fetchUpgradeStatus = async () => {
+ const upgrade_resp = await fetch(`${BASE_API_URL}/synthesis-studio/check-upgrade`, {
+ method: 'GET',
+ });
+ const upgradeStatus = await upgrade_resp.json();
+ return upgradeStatus;
+}
+
+export const useUpgradeStatus = () => {
+ const { data, isLoading, isError, refetch } = useQuery(
+ ["fetchUpgradeStatus", fetchUpgradeStatus],
+ () => fetchUpgradeStatus(),
+ {
+ keepPreviousData: false,
+ refetchInterval: 30000
+ },
+ );
+
+ return {
+ data,
+ isLoading,
+ isError,
+ refetch
+ };
}
\ No newline at end of file
diff --git a/app/client/src/pages/Home/types.ts b/app/client/src/pages/Home/types.ts
index b30dc16..d5c31b2 100644
--- a/app/client/src/pages/Home/types.ts
+++ b/app/client/src/pages/Home/types.ts
@@ -23,4 +23,12 @@ export interface Evaluation {
job_id: string;
job_name: string;
job_status: string;
+}
+
+export interface DatasetDetails {
+ generation: DatasetGeneration;
+}
+
+export interface DatasetGeneration {
+ [key: string]: string;
}
\ No newline at end of file
diff --git a/app/client/src/types.ts b/app/client/src/types.ts
index 4f4ff69..04b3c40 100644
--- a/app/client/src/types.ts
+++ b/app/client/src/types.ts
@@ -5,7 +5,8 @@ export enum Pages {
HOME = 'home',
DATASETS = 'datasets',
WELCOME = 'welcome',
- FEEDBACK = 'feedback'
+ FEEDBACK = 'feedback',
+ UPGRADE = 'upgrade'
}
export enum ModelParameters {
diff --git a/app/core/database.py b/app/core/database.py
index 3d36d0d..3e9ee2b 100644
--- a/app/core/database.py
+++ b/app/core/database.py
@@ -113,6 +113,15 @@ def init_db(self):
job_creator_name TEXT
)
""")
+
+ cursor.execute("""
+ CREATE TABLE IF NOT EXISTS test_metadata (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ timestamp TEXT,
+ display_name TEXT
+
+ )
+ """)
diff --git a/app/main.py b/app/main.py
index a3efd98..1e6bf69 100644
--- a/app/main.py
+++ b/app/main.py
@@ -8,7 +8,9 @@
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse
+from pydantic import BaseModel
from typing import Dict, List, Optional
+import subprocess
import asyncio
from pathlib import Path
from contextlib import asynccontextmanager
@@ -42,6 +44,7 @@
from app.services.model_alignment import ModelAlignment
from app.core.model_handlers import create_handler
from app.services.aws_bedrock import get_bedrock_client
+from app.migrations.alembic_manager import AlembicMigrationManager
#*************Comment this when running locally********************************************
import cmlapi
@@ -73,8 +76,45 @@ def get_total_size(file_paths):
return total_gb
+def restart_application():
+ """Restart the CML application"""
+ try:
+ cml = cmlapi.default_client()
+ project_id = os.getenv("CDSW_PROJECT_ID")
+ apps_list = cml.list_applications(project_id).applications
+ found_app_list = list(filter(lambda app: 'Synthetic Data Studio' in app.name, apps_list))
+
+ if len(found_app_list) > 0:
+ app = found_app_list[0]
+ if app.status == "APPLICATION_RUNNING":
+ try:
+ cml.restart_application(project_id, app.id)
+ except Exception as e:
+ raise (f"Failed to restart application {app.name}: {str(e)}")
+ else:
+ raise ValueError("Synthetic Data Studio application not found")
+
+ except Exception as e:
+ print(f"Error restarting application: {e}")
+ raise
+
#*************Comment this when running locally********************************************
+# Add these models
+class StudioUpgradeStatus(BaseModel):
+ git_local_commit: str
+ git_remote_commit: str
+ updates_available: bool
+
+class StudioUpgradeResponse(BaseModel):
+ success: bool
+ message: str
+ git_updated: bool
+ frontend_rebuilt: bool
+
+
+
+
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Lifespan context manager for the FastAPI application"""
@@ -242,8 +282,16 @@ def get_timeout_for_request(request: Request) -> float:
evaluator_service = EvaluatorService()
export_service = Export_Service()
db_manager = DatabaseManager()
+# Initialize the migration manager
+alembic_manager = AlembicMigrationManager()
+alembic_manager = AlembicMigrationManager("metadata.db")
-
+@app.on_event("startup")
+async def startup_event():
+ """Check for and apply any pending migrations on startup"""
+ success, message = await alembic_manager.handle_database_upgrade()
+ if not success:
+ print(f"Warning: {message}")
@app.post("/get_project_files", include_in_schema=True, responses = responses,
description = "get project file details")
@@ -1186,6 +1234,143 @@ async def get_example_payloads(use_case:UseCase):
return payload
+
+# Add these two endpoints
+@app.get("/synthesis-studio/check-upgrade", response_model=StudioUpgradeStatus)
+async def check_upgrade_status():
+ """Check if any upgrades are available"""
+ try:
+ # Fetch latest changes
+ subprocess.run(["git", "fetch"], check=True, capture_output=True)
+
+ # Get current branch
+ branch = subprocess.run(
+ ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+ check=True,
+ capture_output=True,
+ text=True
+ ).stdout.strip()
+
+ # Get local and remote commits
+ local_commit = subprocess.run(
+ ["git", "rev-parse", branch],
+ check=True,
+ capture_output=True,
+ text=True
+ ).stdout.strip()
+
+ remote_commit = subprocess.run(
+ ["git", "rev-parse", f"origin/{branch}"],
+ check=True,
+ capture_output=True,
+ text=True
+ ).stdout.strip()
+
+ updates_available = local_commit != remote_commit
+
+ return StudioUpgradeStatus(
+ git_local_commit=local_commit,
+ git_remote_commit=remote_commit,
+ updates_available=updates_available
+ )
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/synthesis-studio/upgrade", response_model=StudioUpgradeResponse)
+async def perform_upgrade():
+ """
+ Perform upgrade process:
+ 1. Pull latest code
+ 2. Run database migrations with Alembic
+ 3. Run build_client.sh
+ 4. Run start_application.py
+ 5. Restart CML application
+ """
+ try:
+ messages = []
+ git_updated = False
+ frontend_rebuilt = False
+ db_upgraded = False
+
+ # 1. Git operations
+ try:
+ # Stash any changes
+ subprocess.run(["git", "stash"], check=True, capture_output=True)
+
+ # Pull updates
+ subprocess.run(["git", "pull"], check=True, capture_output=True)
+
+ # Try to pop stash
+ try:
+ subprocess.run(["git", "stash", "pop"], check=True, capture_output=True)
+ except subprocess.CalledProcessError:
+ messages.append("Warning: Could not restore local changes")
+
+ git_updated = True
+ messages.append("Git repository updated")
+
+ except subprocess.CalledProcessError as e:
+ messages.append(f"Git update failed: {e}")
+ raise HTTPException(status_code=500, detail=str(e))
+
+ # 2. Database migrations
+ try:
+ db_success, db_message = await alembic_manager.handle_database_upgrade()
+ if db_success:
+ db_upgraded = True
+ messages.append(db_message)
+ else:
+ messages.append(f"Database upgrade failed: {db_message}")
+ raise HTTPException(status_code=500, detail=db_message)
+ except Exception as e:
+ messages.append(f"Database migration failed: {str(e)}")
+ raise HTTPException(status_code=500, detail=str(e))
+
+ # 3. Run build_client.sh
+ try:
+ script_path = "build/build_client.py"
+ if os.getenv("IS_COMPOSABLE"):
+ script_path = os.path.join('synthetic-data-studio', script_path)
+ subprocess.run(["python", script_path], check=True)
+ frontend_rebuilt = True
+ messages.append("Frontend rebuilt successfully")
+ except subprocess.CalledProcessError as e:
+ messages.append(f"Frontend build failed: {e}")
+ raise HTTPException(status_code=500, detail=str(e))
+
+ # # 4. Run start_application.py
+ # try:
+ # subprocess.run(["python", "build/start_application.py"], check=True)
+ # messages.append("Application start script completed")
+ # except subprocess.CalledProcessError as e:
+ # messages.append(f"Application start script failed: {e}")
+ # raise HTTPException(status_code=500, detail=str(e))
+
+ # 4. Restart CML application
+ if git_updated or frontend_rebuilt or db_upgraded:
+ try:
+ # Small delay to ensure logs are captured
+ time.sleep(10)
+ restart_application()
+ messages.append("Application restart initiated")
+
+ # Note: This response might not reach the client due to the restart
+ return StudioUpgradeResponse(
+ success=True,
+ message="; ".join(messages),
+ git_updated=git_updated,
+ frontend_rebuilt=frontend_rebuilt
+ )
+
+ except Exception as e:
+ messages.append(f"Application restart failed: {e}")
+ raise HTTPException(status_code=500, detail=str(e))
+
+ except Exception as e:
+ raise HTTPException(
+ status_code=500,
+ detail=f"Upgrade failed: {str(e)}"
+ )
#****** comment below for testing just backend**************
current_directory = os.path.dirname(os.path.abspath(__file__))
client_build_path = os.path.join(current_directory, "client", "dist")
diff --git a/app/migrations/alembic_manager.py b/app/migrations/alembic_manager.py
new file mode 100644
index 0000000..51538c6
--- /dev/null
+++ b/app/migrations/alembic_manager.py
@@ -0,0 +1,61 @@
+# app/migrations/alembic_manager.py
+from alembic.config import Config
+from alembic import command
+from alembic.script import ScriptDirectory
+from alembic.runtime.migration import MigrationContext
+from pathlib import Path
+import os
+from sqlalchemy import create_engine
+
+class AlembicMigrationManager:
+ def __init__(self, db_path: str = None):
+ """Initialize Alembic with the same database path as DatabaseManager"""
+ self.app_path = Path(__file__).parent.parent.parent
+
+ if db_path is None:
+ db_path = os.path.join(self.app_path, "metadata.db")
+ self.db_path = db_path
+
+ # Initialize Alembic config
+ self.alembic_cfg = Config(str(self.app_path / "alembic.ini"))
+ self.alembic_cfg.set_main_option('script_location', str(self.app_path / "alembic"))
+ self.alembic_cfg.set_main_option('sqlalchemy.url', f'sqlite:///{db_path}')
+
+ # Create engine for version checks
+ self.engine = create_engine(f'sqlite:///{db_path}')
+
+ async def get_db_version(self) -> str:
+ """Get current database version"""
+ with self.engine.connect() as conn:
+ context = MigrationContext.configure(conn)
+ return context.get_current_revision()
+
+ async def handle_database_upgrade(self) -> tuple[bool, str]:
+ """
+ Handle database migrations carefully to avoid disrupting existing data
+ """
+ try:
+ # First check if alembic_version table exists
+ try:
+ version = await self.get_db_version()
+ if version is None:
+ # Database exists but no alembic version - stamp current
+ command.stamp(self.alembic_cfg, "head")
+ return True, "Existing database stamped with current version"
+ except Exception:
+ # No alembic_version table - stamp current
+ command.stamp(self.alembic_cfg, "head")
+ return True, "Existing database stamped with current version"
+
+ # Now check for and apply any new migrations
+ script = ScriptDirectory.from_config(self.alembic_cfg)
+ head_revision = script.get_current_head()
+
+ if version != head_revision:
+ command.upgrade(self.alembic_cfg, "head")
+ return True, "Database schema updated successfully"
+
+ return True, "Database schema is up to date"
+
+ except Exception as e:
+ return False, f"Error during database upgrade: {str(e)}"
\ No newline at end of file
diff --git a/app/migrations/alembic_schema_models.py b/app/migrations/alembic_schema_models.py
new file mode 100644
index 0000000..4035467
--- /dev/null
+++ b/app/migrations/alembic_schema_models.py
@@ -0,0 +1,79 @@
+# app/migrations/alembic_schema_models.py
+from sqlalchemy import create_engine, Column, Integer, Text, Float, MetaData
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()
+
+class GenerationMetadataModel(Base):
+ __tablename__ = 'generation_metadata'
+
+ id = Column(Integer, primary_key=True, autoincrement=True)
+ timestamp = Column(Text)
+ technique = Column(Text)
+ model_id = Column(Text)
+ inference_type = Column(Text)
+ caii_endpoint = Column(Text)
+ use_case = Column(Text)
+ custom_prompt = Column(Text)
+ model_parameters = Column(Text)
+ input_key = Column(Text)
+ output_key = Column(Text)
+ output_value = Column(Text)
+ generate_file_name = Column(Text, unique=True)
+ display_name = Column(Text)
+ local_export_path = Column(Text)
+ hf_export_path = Column(Text)
+ num_questions = Column(Float)
+ total_count = Column(Float)
+ topics = Column(Text)
+ examples = Column(Text)
+ schema = Column(Text)
+ doc_paths = Column(Text)
+ input_path = Column(Text)
+ job_id = Column(Text)
+ job_name = Column(Text, unique=True)
+ job_status = Column(Text)
+ job_creator_name = Column(Text)
+
+class EvaluationMetadataModel(Base):
+ __tablename__ = 'evaluation_metadata'
+
+ id = Column(Integer, primary_key=True, autoincrement=True)
+ timestamp = Column(Text)
+ model_id = Column(Text)
+ inference_type = Column(Text)
+ caii_endpoint = Column(Text)
+ use_case = Column(Text)
+ custom_prompt = Column(Text)
+ model_parameters = Column(Text)
+ generate_file_name = Column(Text)
+ evaluate_file_name = Column(Text, unique=True)
+ display_name = Column(Text)
+ local_export_path = Column(Text)
+ examples = Column(Text)
+ average_score = Column(Float)
+ job_id = Column(Text)
+ job_name = Column(Text, unique=True)
+ job_status = Column(Text)
+ job_creator_name = Column(Text)
+
+class ExportMetadataModel(Base):
+ __tablename__ = 'export_metadata'
+
+ id = Column(Integer, primary_key=True, autoincrement=True)
+ timestamp = Column(Text)
+ display_export_name = Column(Text)
+ display_name = Column(Text)
+ local_export_path = Column(Text)
+ hf_export_path = Column(Text)
+ job_id = Column(Text)
+ job_name = Column(Text, unique=True)
+ job_status = Column(Text)
+ job_creator_name = Column(Text)
+
+class TestMetadataModel(Base):
+ __tablename__ = 'test_metadata'
+ id = Column(Integer, primary_key=True, autoincrement=True)
+ timestamp = Column(Text)
+ display_name = Column(Text)
+
\ No newline at end of file
diff --git a/app/run.py b/app/run.py
index a42bd7a..d93e604 100644
--- a/app/run.py
+++ b/app/run.py
@@ -11,4 +11,3 @@
port=port,
reload=True
)
-
diff --git a/docs/guides/evaluation_workflow.md b/docs/guides/evaluation_workflow.md
new file mode 100644
index 0000000..a6396b4
--- /dev/null
+++ b/docs/guides/evaluation_workflow.md
@@ -0,0 +1,100 @@
+# Supervised Finetuning Workflow:
+
+In this workflow we will see how we can evaluate synthetic data generated in previous steps using Large Language Model as a judge.
+
+User can Trigger evaluation via List view where they can chose evaluation to begin from the dropdown.
+
+
+
+
+## Evaluation Workflow
+
+Similar to generation evaluation also allows users to specify following
+
+1. #### Display Name
+2. #### Model Provider: AWS Bedrock or Cloudera AI Inference
+3. #### Model ID: Calude , LLAMA , Mistral etc.
+
+The code generation and text2sql are templates which allow users to select from already curated prompts, and examples to evaluate datasets.
+
+Custom template on the other hand allows users to define everything from scratch to evaluate created synthteic dataset from previous step.
+
+The screen shot for the same can be seen below:
+
+
+
+### Prompt and Model Parameters
+
+#### Prompt:
+This step allows user to curate their prompts manuallly, or chose from given templates or let LLM curate a prompt based on their description of use case.
+
+```json
+{
+"""Below is a Python coding Question and Solution pair generated by an LLM. Evaluate its quality as a Senior Developer would, considering its suitability for professional use. Use the additive 5-point scoring system described below.
+
+Points are accumulated based on the satisfaction of each criterion:
+ 1. Add 1 point if the code implements basic functionality and solves the core problem, even if it includes some minor issues or non-optimal approaches.
+ 2. Add another point if the implementation is generally correct but lacks refinement in style or fails to follow some best practices. It might use inconsistent naming conventions or have occasional inefficiencies.
+ 3. Award a third point if the code is appropriate for professional use and accurately implements the required functionality. It demonstrates good understanding of Python concepts and common patterns, though it may not be optimal. It resembles the work of a competent developer but may have room for improvement in efficiency or organization.
+ 4. Grant a fourth point if the code is highly efficient and follows Python best practices, exhibiting consistent style and appropriate documentation. It could be similar to the work of an experienced developer, offering robust error handling, proper type hints, and effective use of built-in features. The result is maintainable, well-structured, and valuable for production use.
+ 5. Bestow a fifth point if the code is outstanding, demonstrating mastery of Python and software engineering principles. It includes comprehensive error handling, efficient algorithms, proper testing considerations, and excellent documentation. The solution is scalable, performant, and shows attention to edge cases and security considerations."""
+}
+```
+
+
+#### Model Parameters
+
+We let user decide on following model Parameters:
+
+- **Temperature**
+- **TopK**
+- **TopP**
+
+
+
+### Examples:
+
+In the next step user can specify examples they would want to give for their evaluation of dataset so that LLM can follow same format and Judge/rate datasets accordingly.
+
+The examples for evaluation would be like following:
+
+The **scoring** and **Justification** can be defined by user within the prompt and example for example in this use case we use a 5 point rating system, user can make it **10 point ratings**, **Boolean** , **subjective ("bad", "Good", "Average")** etc.
+
+```json
+{
+ "score": 3,
+ "justification": """The code achieves 3 points by implementing core functionality correctly (1),
+ showing generally correct implementation with proper syntax (2),
+ and being suitable for professional use with good Python patterns and accurate functionality (3).
+ While it demonstrates competent development practices, it lacks the robust error handling
+ and type hints needed for point 4, and could benefit from better efficiency optimization and code organization."""
+ },
+ {
+ "score": 4,
+ "justification": """
+ The code earns 4 points by implementing basic functionality (1), showing correct implementation (2),
+ being production-ready (3), and demonstrating high efficiency with Python best practices
+ including proper error handling, type hints, and clear documentation (4).
+ It exhibits experienced developer qualities with well-structured code and maintainable design, though
+ it lacks the comprehensive testing and security considerations needed for a perfect score."""
+}
+```
+
+
+
+### Final Output:
+
+Finally user can see how their output looks like with corresponding Scores and Justifications.
+
+The output will be saved in Project File System within Cloudera environment.
+
+
+
+The output and corresponding metadata (scores,model etc.) can be seen on the **Evaluations** list view as well as shown in screen shot below.
+
+
+
+
+
+
+
diff --git a/docs/guides/screenshots/evaluate_home_page.png b/docs/guides/screenshots/evaluate_home_page.png
new file mode 100644
index 0000000..501d105
Binary files /dev/null and b/docs/guides/screenshots/evaluate_home_page.png differ
diff --git a/docs/guides/screenshots/evaluate_list.png b/docs/guides/screenshots/evaluate_list.png
new file mode 100644
index 0000000..caf1635
Binary files /dev/null and b/docs/guides/screenshots/evaluate_list.png differ
diff --git a/docs/guides/screenshots/evaluate_output.png b/docs/guides/screenshots/evaluate_output.png
new file mode 100644
index 0000000..655898c
Binary files /dev/null and b/docs/guides/screenshots/evaluate_output.png differ
diff --git a/docs/guides/screenshots/evaluation_sds.png b/docs/guides/screenshots/evaluation_sds.png
new file mode 100644
index 0000000..4167cbc
Binary files /dev/null and b/docs/guides/screenshots/evaluation_sds.png differ
diff --git a/docs/guides/screenshots/export_list.png b/docs/guides/screenshots/export_list.png
new file mode 100644
index 0000000..6e0d079
Binary files /dev/null and b/docs/guides/screenshots/export_list.png differ
diff --git a/docs/guides/screenshots/sds_examples.png b/docs/guides/screenshots/sds_examples.png
new file mode 100644
index 0000000..cc84918
Binary files /dev/null and b/docs/guides/screenshots/sds_examples.png differ
diff --git a/docs/guides/screenshots/sds_export.png b/docs/guides/screenshots/sds_export.png
new file mode 100644
index 0000000..5ecc272
Binary files /dev/null and b/docs/guides/screenshots/sds_export.png differ
diff --git a/docs/guides/screenshots/sds_generation.png b/docs/guides/screenshots/sds_generation.png
new file mode 100644
index 0000000..6f3aaa1
Binary files /dev/null and b/docs/guides/screenshots/sds_generation.png differ
diff --git a/docs/guides/screenshots/sds_hf_export.png b/docs/guides/screenshots/sds_hf_export.png
new file mode 100644
index 0000000..825c381
Binary files /dev/null and b/docs/guides/screenshots/sds_hf_export.png differ
diff --git a/docs/guides/screenshots/sds_home_page.png b/docs/guides/screenshots/sds_home_page.png
new file mode 100644
index 0000000..c5215c5
Binary files /dev/null and b/docs/guides/screenshots/sds_home_page.png differ
diff --git a/docs/guides/screenshots/sds_output.png b/docs/guides/screenshots/sds_output.png
new file mode 100644
index 0000000..42dfc2f
Binary files /dev/null and b/docs/guides/screenshots/sds_output.png differ
diff --git a/docs/guides/screenshots/sds_prompt.png b/docs/guides/screenshots/sds_prompt.png
new file mode 100644
index 0000000..9712460
Binary files /dev/null and b/docs/guides/screenshots/sds_prompt.png differ
diff --git a/docs/guides/screenshots/sds_summary.png b/docs/guides/screenshots/sds_summary.png
new file mode 100644
index 0000000..9632a7e
Binary files /dev/null and b/docs/guides/screenshots/sds_summary.png differ
diff --git a/docs/guides/sft_workflow.md b/docs/guides/sft_workflow.md
new file mode 100644
index 0000000..c360d28
--- /dev/null
+++ b/docs/guides/sft_workflow.md
@@ -0,0 +1,116 @@
+# Generation Workflow:
+
+In this workflow we will see how we can create synthetic data for finetuning our models. The users in this workflow can chose from provided templates like.
+
+## Templates
+
+1. **Code Generation**
+2. **Text to SQL**
+3. **Custom**
+
+The code generation and text2sql are templates which allow users to select from already curated prompts, seeds(more on it below) and examples to produce datasets.
+
+Custom template on the other hand allows users to define everything from scratch and create synthteic dataset for their custom Enterprise use cases.
+
+## Workflow Example: Code Generation
+
+### Home Page
+On home Page user can click on Create Datasets to Get Started
+
+
+
+### Generate Configuration: In the next step user gets to specify following fields:
+
+1. #### Display Name
+2. #### Model Provider: AWS Bedrock or Cloudera AI Inference
+3. #### Model ID: Calude , LLAMA , Mistral etc.
+4. #### Workflow
+ a. Supervised Finetuning :- Generate Prompt and Completion Pairs with or without documents(pdfs, docs, txt etc.)
+ b. Custom Data Curation:- Use Input as json array(which can be uploaded) from the user and generate response based on that. In this case user can have their own inputs, instructions and get customised generated output for corresponding input.
+5. #### Files: Input Files user can chose from their project file system for above workflows
+
+
+
+### Prompt and Model Parameters
+
+#### Prompt:
+This step allows user to curate their prompts manuallly, or chose from given templates or let LLM curate a prompt based on their description of use case.
+
+```json
+{
+"""Write a programming question-pair for the following topic:
+
+Requirements:
+- Each solution must include working code examples
+- Include explanations with the code
+- Follow the same format as the examples
+- Ensure code is properly formatted with appropriate indentation"""
+}
+```
+
+#### Seeds:
+
+This helps LLM diversify dataset user wants to generate. We drew inspiration from **[Self Intruct Paper](https://huggingface.co/papers/2212.10560)**
+ , where 175 hand crafted human seed instructions were used to diversify curation of dataset.
+
+ For example, for code generation, seeds can be:
+- **Algorithms for Operation Research**
+- **Web Development with Flask**
+- **PyTorch for Reinforcement Learning**
+
+Similarly for language translation, seeds can be:
+- **Poems**
+- **Greetings in Formal Communication**
+- **Haikus**
+
+#### Model Parameters
+
+We let user decide on following model Parameters:
+
+- **Temperature**
+- **TopK**
+- **TopP**
+
+#### Dataset Size
+
+
+
+### Examples:
+
+In the next step user can specify examples they would want to give for their synthetic dataset generation so that LLM can follow same format and create datasets accordingly.
+
+The examples for code geneartion would be like following:
+
+```json
+{
+ "question": "How do you read a CSV file into a pandas DataFrame?",
+ "solution": """You can use pandas.read_csv(). Here's an example
+
+ import pandas as pd
+ df = pd.read_csv('data.csv')
+ print(df.head())
+ print(df.info())
+"""
+}
+```
+
+
+
+
+### Summary:
+
+This allows user to finally look at prompt, seeds, dataset size and other parameters they have selected for data generation.
+
+
+
+### Final Output:
+
+Finally user can see how their output looks like with corresponding Prompts and Completions.
+
+The output will be saved in Project File System within Cloudera environment.
+
+
+
+
+
+The output and corresponding metadata (scores,model etc.) can be seen on the **Generations** list view as well as shown in screen shot below.
diff --git a/docs/technical_overview.md b/docs/technical_overview.md
index 7c7027c..abf9bdf 100644
--- a/docs/technical_overview.md
+++ b/docs/technical_overview.md
@@ -165,7 +165,7 @@ df = pd.read_csv('data.csv')\n
"""}]
-Write a programming question-pair for the following topic:
+Write a programming question-answer pair for the following topic:
Requirements:
- Each solution must include working code examples
- Include explanations with the code
diff --git a/requirements.txt b/requirements.txt
index e7ea7f6..0d98703 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,6 +11,8 @@ PyMuPDF==1.25.1
python-docx==1.1.2
#better-profanity==0.7.0
#guardrails-ai==0.6.1
+alembic==1.14.1
+sqlalchemy==2.0.38
# AWS SDK
boto3==1.35.48
+
+
+## Evaluation Workflow
+
+Similar to generation evaluation also allows users to specify following
+
+1. #### Display Name
+2. #### Model Provider: AWS Bedrock or Cloudera AI Inference
+3. #### Model ID: Calude , LLAMA , Mistral etc.
+
+The code generation and text2sql are templates which allow users to select from already curated prompts, and examples to evaluate datasets.
+
+Custom template on the other hand allows users to define everything from scratch to evaluate created synthteic dataset from previous step.
+
+The screen shot for the same can be seen below:
+
+
+
+### Prompt and Model Parameters
+
+#### Prompt:
+This step allows user to curate their prompts manuallly, or chose from given templates or let LLM curate a prompt based on their description of use case.
+
+```json
+{
+"""Below is a Python coding Question and Solution pair generated by an LLM. Evaluate its quality as a Senior Developer would, considering its suitability for professional use. Use the additive 5-point scoring system described below.
+
+Points are accumulated based on the satisfaction of each criterion:
+ 1. Add 1 point if the code implements basic functionality and solves the core problem, even if it includes some minor issues or non-optimal approaches.
+ 2. Add another point if the implementation is generally correct but lacks refinement in style or fails to follow some best practices. It might use inconsistent naming conventions or have occasional inefficiencies.
+ 3. Award a third point if the code is appropriate for professional use and accurately implements the required functionality. It demonstrates good understanding of Python concepts and common patterns, though it may not be optimal. It resembles the work of a competent developer but may have room for improvement in efficiency or organization.
+ 4. Grant a fourth point if the code is highly efficient and follows Python best practices, exhibiting consistent style and appropriate documentation. It could be similar to the work of an experienced developer, offering robust error handling, proper type hints, and effective use of built-in features. The result is maintainable, well-structured, and valuable for production use.
+ 5. Bestow a fifth point if the code is outstanding, demonstrating mastery of Python and software engineering principles. It includes comprehensive error handling, efficient algorithms, proper testing considerations, and excellent documentation. The solution is scalable, performant, and shows attention to edge cases and security considerations."""
+}
+```
+
+
+#### Model Parameters
+
+We let user decide on following model Parameters:
+
+- **Temperature**
+- **TopK**
+- **TopP**
+
+
+
+### Examples:
+
+In the next step user can specify examples they would want to give for their evaluation of dataset so that LLM can follow same format and Judge/rate datasets accordingly.
+
+The examples for evaluation would be like following:
+
+The **scoring** and **Justification** can be defined by user within the prompt and example for example in this use case we use a 5 point rating system, user can make it **10 point ratings**, **Boolean** , **subjective ("bad", "Good", "Average")** etc.
+
+```json
+{
+ "score": 3,
+ "justification": """The code achieves 3 points by implementing core functionality correctly (1),
+ showing generally correct implementation with proper syntax (2),
+ and being suitable for professional use with good Python patterns and accurate functionality (3).
+ While it demonstrates competent development practices, it lacks the robust error handling
+ and type hints needed for point 4, and could benefit from better efficiency optimization and code organization."""
+ },
+ {
+ "score": 4,
+ "justification": """
+ The code earns 4 points by implementing basic functionality (1), showing correct implementation (2),
+ being production-ready (3), and demonstrating high efficiency with Python best practices
+ including proper error handling, type hints, and clear documentation (4).
+ It exhibits experienced developer qualities with well-structured code and maintainable design, though
+ it lacks the comprehensive testing and security considerations needed for a perfect score."""
+}
+```
+
+
+
+### Final Output:
+
+Finally user can see how their output looks like with corresponding Scores and Justifications.
+
+The output will be saved in Project File System within Cloudera environment.
+
+
+
+The output and corresponding metadata (scores,model etc.) can be seen on the **Evaluations** list view as well as shown in screen shot below.
+
+
+
+
+
+
+
diff --git a/docs/guides/screenshots/evaluate_home_page.png b/docs/guides/screenshots/evaluate_home_page.png
new file mode 100644
index 0000000..501d105
Binary files /dev/null and b/docs/guides/screenshots/evaluate_home_page.png differ
diff --git a/docs/guides/screenshots/evaluate_list.png b/docs/guides/screenshots/evaluate_list.png
new file mode 100644
index 0000000..caf1635
Binary files /dev/null and b/docs/guides/screenshots/evaluate_list.png differ
diff --git a/docs/guides/screenshots/evaluate_output.png b/docs/guides/screenshots/evaluate_output.png
new file mode 100644
index 0000000..655898c
Binary files /dev/null and b/docs/guides/screenshots/evaluate_output.png differ
diff --git a/docs/guides/screenshots/evaluation_sds.png b/docs/guides/screenshots/evaluation_sds.png
new file mode 100644
index 0000000..4167cbc
Binary files /dev/null and b/docs/guides/screenshots/evaluation_sds.png differ
diff --git a/docs/guides/screenshots/export_list.png b/docs/guides/screenshots/export_list.png
new file mode 100644
index 0000000..6e0d079
Binary files /dev/null and b/docs/guides/screenshots/export_list.png differ
diff --git a/docs/guides/screenshots/sds_examples.png b/docs/guides/screenshots/sds_examples.png
new file mode 100644
index 0000000..cc84918
Binary files /dev/null and b/docs/guides/screenshots/sds_examples.png differ
diff --git a/docs/guides/screenshots/sds_export.png b/docs/guides/screenshots/sds_export.png
new file mode 100644
index 0000000..5ecc272
Binary files /dev/null and b/docs/guides/screenshots/sds_export.png differ
diff --git a/docs/guides/screenshots/sds_generation.png b/docs/guides/screenshots/sds_generation.png
new file mode 100644
index 0000000..6f3aaa1
Binary files /dev/null and b/docs/guides/screenshots/sds_generation.png differ
diff --git a/docs/guides/screenshots/sds_hf_export.png b/docs/guides/screenshots/sds_hf_export.png
new file mode 100644
index 0000000..825c381
Binary files /dev/null and b/docs/guides/screenshots/sds_hf_export.png differ
diff --git a/docs/guides/screenshots/sds_home_page.png b/docs/guides/screenshots/sds_home_page.png
new file mode 100644
index 0000000..c5215c5
Binary files /dev/null and b/docs/guides/screenshots/sds_home_page.png differ
diff --git a/docs/guides/screenshots/sds_output.png b/docs/guides/screenshots/sds_output.png
new file mode 100644
index 0000000..42dfc2f
Binary files /dev/null and b/docs/guides/screenshots/sds_output.png differ
diff --git a/docs/guides/screenshots/sds_prompt.png b/docs/guides/screenshots/sds_prompt.png
new file mode 100644
index 0000000..9712460
Binary files /dev/null and b/docs/guides/screenshots/sds_prompt.png differ
diff --git a/docs/guides/screenshots/sds_summary.png b/docs/guides/screenshots/sds_summary.png
new file mode 100644
index 0000000..9632a7e
Binary files /dev/null and b/docs/guides/screenshots/sds_summary.png differ
diff --git a/docs/guides/sft_workflow.md b/docs/guides/sft_workflow.md
new file mode 100644
index 0000000..c360d28
--- /dev/null
+++ b/docs/guides/sft_workflow.md
@@ -0,0 +1,116 @@
+# Generation Workflow:
+
+In this workflow we will see how we can create synthetic data for finetuning our models. The users in this workflow can chose from provided templates like.
+
+## Templates
+
+1. **Code Generation**
+2. **Text to SQL**
+3. **Custom**
+
+The code generation and text2sql are templates which allow users to select from already curated prompts, seeds(more on it below) and examples to produce datasets.
+
+Custom template on the other hand allows users to define everything from scratch and create synthteic dataset for their custom Enterprise use cases.
+
+## Workflow Example: Code Generation
+
+### Home Page
+On home Page user can click on Create Datasets to Get Started
+
+
+
+### Generate Configuration: In the next step user gets to specify following fields:
+
+1. #### Display Name
+2. #### Model Provider: AWS Bedrock or Cloudera AI Inference
+3. #### Model ID: Calude , LLAMA , Mistral etc.
+4. #### Workflow
+ a. Supervised Finetuning :- Generate Prompt and Completion Pairs with or without documents(pdfs, docs, txt etc.)
+ b. Custom Data Curation:- Use Input as json array(which can be uploaded) from the user and generate response based on that. In this case user can have their own inputs, instructions and get customised generated output for corresponding input.
+5. #### Files: Input Files user can chose from their project file system for above workflows
+
+
+
+### Prompt and Model Parameters
+
+#### Prompt:
+This step allows user to curate their prompts manuallly, or chose from given templates or let LLM curate a prompt based on their description of use case.
+
+```json
+{
+"""Write a programming question-pair for the following topic:
+
+Requirements:
+- Each solution must include working code examples
+- Include explanations with the code
+- Follow the same format as the examples
+- Ensure code is properly formatted with appropriate indentation"""
+}
+```
+
+#### Seeds:
+
+This helps LLM diversify dataset user wants to generate. We drew inspiration from **[Self Intruct Paper](https://huggingface.co/papers/2212.10560)**
+ , where 175 hand crafted human seed instructions were used to diversify curation of dataset.
+
+ For example, for code generation, seeds can be:
+- **Algorithms for Operation Research**
+- **Web Development with Flask**
+- **PyTorch for Reinforcement Learning**
+
+Similarly for language translation, seeds can be:
+- **Poems**
+- **Greetings in Formal Communication**
+- **Haikus**
+
+#### Model Parameters
+
+We let user decide on following model Parameters:
+
+- **Temperature**
+- **TopK**
+- **TopP**
+
+#### Dataset Size
+
+
+
+### Examples:
+
+In the next step user can specify examples they would want to give for their synthetic dataset generation so that LLM can follow same format and create datasets accordingly.
+
+The examples for code geneartion would be like following:
+
+```json
+{
+ "question": "How do you read a CSV file into a pandas DataFrame?",
+ "solution": """You can use pandas.read_csv(). Here's an example
+
+ import pandas as pd
+ df = pd.read_csv('data.csv')
+ print(df.head())
+ print(df.info())
+"""
+}
+```
+
+
+
+
+### Summary:
+
+This allows user to finally look at prompt, seeds, dataset size and other parameters they have selected for data generation.
+
+
+
+### Final Output:
+
+Finally user can see how their output looks like with corresponding Prompts and Completions.
+
+The output will be saved in Project File System within Cloudera environment.
+
+
+
+
+
+The output and corresponding metadata (scores,model etc.) can be seen on the **Generations** list view as well as shown in screen shot below.
diff --git a/docs/technical_overview.md b/docs/technical_overview.md
index 7c7027c..abf9bdf 100644
--- a/docs/technical_overview.md
+++ b/docs/technical_overview.md
@@ -165,7 +165,7 @@ df = pd.read_csv('data.csv')\n
"""}]
-Write a programming question-pair for the following topic:
+Write a programming question-answer pair for the following topic:
Requirements:
- Each solution must include working code examples
- Include explanations with the code
diff --git a/requirements.txt b/requirements.txt
index e7ea7f6..0d98703 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,6 +11,8 @@ PyMuPDF==1.25.1
python-docx==1.1.2
#better-profanity==0.7.0
#guardrails-ai==0.6.1
+alembic==1.14.1
+sqlalchemy==2.0.38
# AWS SDK
boto3==1.35.48