mindchain commited on
Commit
d8bc360
·
verified ·
1 Parent(s): 8a7ddca

Upload src/App.jsx with huggingface_hub

Browse files
Files changed (1) hide show
  1. src/App.jsx +643 -375
src/App.jsx CHANGED
@@ -17,13 +17,22 @@ import {
17
  ChevronDown,
18
  ChevronUp,
19
  Copy,
20
- ExternalLink
 
 
 
 
 
 
 
 
21
  } from 'lucide-react'
22
 
23
  function App() {
24
  const [activePhase, setActivePhase] = useState(0)
25
  const [expandedSections, setExpandedSections] = useState({})
26
  const [copiedCode, setCopiedCode] = useState(null)
 
27
 
28
  const toggleSection = (section) => {
29
  setExpandedSections(prev => ({
@@ -41,16 +50,26 @@ function App() {
41
  const phases = [
42
  {
43
  id: 0,
44
- name: 'Phase 0: Seed Generator',
45
- icon: <Zap className="w-6 h-6" />,
 
46
  space: 'mindchain/hf-space-nemo-seed-generator-huggingface-inference',
47
- description: 'Generates initial seed Q&A pairs from a topic using LLM',
48
- color: 'from-yellow-500 to-orange-500',
 
 
49
  endpoint: '/generate',
50
  method: 'POST',
 
 
 
 
 
 
 
51
  requestExample: `{
52
  "topic": "Python programming basics",
53
- "num_seeds": 5,
54
  "model": "Qwen/Qwen2.5-7B-Instruct",
55
  "provider": "together"
56
  }`,
@@ -59,22 +78,34 @@ function App() {
59
  "seeds": [
60
  {
61
  "instruction": "What is a variable in Python?",
62
- "output": "A variable in Python is a named location...",
63
  "quality_score": 8
64
  }
65
  ],
66
- "count": 5
67
  }`
68
  },
69
  {
70
  id: 1,
71
- name: 'Phase 1: Distilabel Generator',
72
- icon: <Cpu className="w-6 h-6" />,
 
73
  space: 'mindchain/hf-space-distilabel-generator-huggingface-inference',
74
- description: 'Generates synthetic data with LLM Judge scoring using UltraFeedback criteria',
75
- color: 'from-blue-500 to-cyan-500',
 
 
76
  endpoint: '/generate',
77
  method: 'POST',
 
 
 
 
 
 
 
 
 
78
  requestExample: `{
79
  "seed_dataset": "mindchain/synthetic-seeds",
80
  "num_records": 100,
@@ -89,8 +120,8 @@ function App() {
89
  "success": true,
90
  "records": [
91
  {
92
- "instruction": "...",
93
- "output": "...",
94
  "quality_score": 8,
95
  "distilabel_metadata": {...}
96
  }
@@ -104,13 +135,24 @@ function App() {
104
  },
105
  {
106
  id: 2,
107
- name: 'Phase 2: Argilla Curator',
108
- icon: <Filter className="w-6 h-6" />,
 
109
  space: 'mindchain/hf-space-argilla-curator-huggingface-inference',
110
- description: 'Final curation with quality scoring and filtering for high-quality dataset',
111
- color: 'from-purple-500 to-pink-500',
 
 
112
  endpoint: '/curate',
113
  method: 'POST',
 
 
 
 
 
 
 
 
114
  requestExample: `{
115
  "raw_dataset": "mindchain/synthetic-distilabel-raw",
116
  "model": "Qwen/Qwen2.5-7B-Instruct",
@@ -124,425 +166,647 @@ function App() {
124
  "total_count": 100,
125
  "filtered_count": 22,
126
  "score_distribution": {
127
- "7": 25,
128
- "8": 30,
129
- "9": 15,
130
- "10": 8
131
  }
132
  }`
133
  }
134
  ]
135
 
136
- const curlExamples = phases.map(phase => `curl -X POST https://${phase.space}.hf.space${phase.endpoint} \\
137
  -H "Content-Type: application/json" \\
138
- -d '${phase.requestExample.split('\n').slice(0, 6).join('\n ')}...'`)
 
 
 
 
 
139
 
140
  return (
141
- <div className="min-h-screen bg-gray-900 text-gray-100">
142
- {/* Header */}
143
- <header className="bg-gradient-to-r from-gray-800 to-gray-900 border-b border-gray-700">
144
- <div className="max-w-7xl mx-auto px-4 py-6">
145
- <div className="flex items-center gap-4">
146
- <div className="p-3 bg-gradient-to-br from-indigo-500 to-purple-600 rounded-xl">
147
- <Database className="w-8 h-8" />
 
 
 
 
 
 
 
 
 
148
  </div>
149
- <div>
150
- <h1 className="text-2xl font-bold">Synthetic Data Pipeline</h1>
151
- <p className="text-gray-400">HuggingFace Spaces Documentation</p>
 
 
 
 
 
 
 
 
 
 
 
 
 
152
  </div>
 
 
 
 
 
 
 
 
153
  </div>
154
  </div>
155
- </header>
156
-
157
- {/* Main Content */}
158
- <main className="max-w-7xl mx-auto px-4 py-8">
159
- {/* Overview Section */}
160
- <section className="mb-12">
161
- <div className="flex items-center gap-2 mb-4">
162
- <BookOpen className="w-5 h-5 text-indigo-400" />
163
- <h2 className="text-xl font-semibold">Overview</h2>
 
164
  </div>
165
- <div className="bg-gray-800 rounded-xl p-6 border border-gray-700">
166
- <p className="text-gray-300 mb-4">
167
- This pipeline generates high-quality synthetic Q&A datasets using HuggingFace Spaces and the HF Inference API.
168
- It follows a three-phase approach: seed generation, distilabel-based augmentation with LLM Judge scoring,
169
- and final curation.
170
- </p>
171
- <div className="grid grid-cols-1 md:grid-cols-3 gap-4 mt-6">
172
- <div className="bg-gray-700/50 rounded-lg p-4 flex items-center gap-3">
173
- <CheckCircle className="w-5 h-5 text-green-400" />
174
- <span className="text-sm">CPU-only (no GPU required)</span>
175
- </div>
176
- <div className="bg-gray-700/50 rounded-lg p-4 flex items-center gap-3">
177
- <CheckCircle className="w-5 h-5 text-green-400" />
178
- <span className="text-sm">HF Inference API via Together</span>
179
- </div>
180
- <div className="bg-gray-700/50 rounded-lg p-4 flex items-center gap-3">
181
- <CheckCircle className="w-5 h-5 text-green-400" />
182
- <span className="text-sm">Qwen2.5-7B-Instruct model</span>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
183
  </div>
 
 
 
 
 
 
 
 
 
 
 
184
  </div>
 
185
  </div>
186
- </section>
187
 
188
- {/* Pipeline Architecture */}
189
- <section className="mb-12">
190
- <div className="flex items-center gap-2 mb-4">
191
- <Layers className="w-5 h-5 text-indigo-400" />
192
- <h2 className="text-xl font-semibold">Pipeline Architecture</h2>
193
- </div>
194
- <div className="bg-gray-800 rounded-xl p-6 border border-gray-700">
195
- <div className="flex flex-col md:flex-row items-center justify-center gap-4">
196
- {phases.map((phase, index) => (
197
- <React.Fragment key={phase.id}>
198
- <button
199
- onClick={() => setActivePhase(phase.id)}
200
- className={`flex-1 w-full p-4 rounded-xl border-2 transition-all ${
201
- activePhase === phase.id
202
- ? 'border-indigo-500 bg-indigo-500/10'
203
- : 'border-gray-600 hover:border-gray-500'
204
- }`}
205
- >
206
- <div className={`inline-flex p-2 rounded-lg bg-gradient-to-r ${phase.color} mb-2`}>
207
- {phase.icon}
208
  </div>
209
- <h3 className="font-semibold text-sm">{phase.name}</h3>
210
- </button>
211
- {index < phases.length - 1 && (
212
- <ArrowRight className="w-6 h-6 text-gray-500 hidden md:block" />
213
- )}
214
- </React.Fragment>
215
  ))}
216
  </div>
217
  </div>
218
- </section>
219
-
220
- {/* Phase Details */}
221
- <section className="mb-12">
222
- <div className="bg-gray-800 rounded-xl border border-gray-700 overflow-hidden">
223
- <div className={`p-6 bg-gradient-to-r ${phases[activePhase].color}`}>
224
- <div className="flex items-center gap-3">
225
- {phases[activePhase].icon}
226
- <h2 className="text-xl font-bold">{phases[activePhase].name}</h2>
227
- </div>
228
- <p className="mt-2 opacity-90">{phases[activePhase].description}</p>
229
  </div>
 
 
230
 
231
- <div className="p-6 space-y-6">
232
- {/* Space URL */}
233
- <div>
234
- <div className="flex items-center gap-2 mb-2">
235
- <ExternalLink className="w-4 h-4 text-gray-400" />
236
- <span className="text-sm font-medium text-gray-400">HuggingFace Space</span>
237
- </div>
238
- <a
239
- href={`https://huggingface.co/spaces/${phases[activePhase].space}`}
240
- target="_blank"
241
- rel="noopener noreferrer"
242
- className="text-indigo-400 hover:text-indigo-300 text-sm font-mono"
243
  >
244
- {phases[activePhase].space}
245
- </a>
246
- </div>
247
-
248
- {/* API Endpoint */}
249
- <div>
250
- <div className="flex items-center gap-2 mb-2">
251
- <Terminal className="w-4 h-4 text-gray-400" />
252
- <span className="text-sm font-medium text-gray-400">API Endpoint</span>
253
- <span className="px-2 py-0.5 bg-green-500/20 text-green-400 text-xs rounded">{phases[activePhase].method}</span>
254
- </div>
255
- <code className="block bg-gray-900 rounded-lg p-3 text-sm font-mono text-gray-300">
256
- https://{phases[activePhase].space}.hf.space{phases[activePhase].endpoint}
257
- </code>
258
- </div>
259
 
260
- {/* Request Example */}
261
- <div>
262
- <div className="flex items-center justify-between mb-2">
263
- <div className="flex items-center gap-2">
264
- <Code className="w-4 h-4 text-gray-400" />
265
- <span className="text-sm font-medium text-gray-400">Request Body</span>
 
266
  </div>
267
- <button
268
- onClick={() => copyToClipboard(phases[activePhase].requestExample, `req-${activePhase}`)}
269
- className="text-gray-400 hover:text-white"
270
- >
271
- {copiedCode === `req-${activePhase}` ? (
272
- <CheckCircle className="w-4 h-4 text-green-400" />
273
- ) : (
274
- <Copy className="w-4 h-4" />
275
- )}
276
- </button>
277
  </div>
278
- <pre className="bg-gray-900 rounded-lg p-4 text-sm font-mono text-gray-300 overflow-x-auto">
279
- {phases[activePhase].requestExample}
280
- </pre>
281
- </div>
282
 
283
- {/* Response Example */}
284
- <div>
285
- <div className="flex items-center justify-between mb-2">
286
- <div className="flex items-center gap-2">
287
- <Code className="w-4 h-4 text-gray-400" />
288
- <span className="text-sm font-medium text-gray-400">Response Example</span>
 
 
289
  </div>
290
- <button
291
- onClick={() => copyToClipboard(phases[activePhase].responseExample, `res-${activePhase}`)}
292
- className="text-gray-400 hover:text-white"
293
- >
294
- {copiedCode === `res-${activePhase}` ? (
295
- <CheckCircle className="w-4 h-4 text-green-400" />
296
- ) : (
297
- <Copy className="w-4 h-4" />
298
- )}
299
- </button>
300
- </div>
301
- <pre className="bg-gray-900 rounded-lg p-4 text-sm font-mono text-gray-300 overflow-x-auto">
302
- {phases[activePhase].responseExample}
303
- </pre>
304
- </div>
305
 
306
- {/* cURL Command */}
307
- <div>
308
- <div className="flex items-center justify-between mb-2">
309
  <div className="flex items-center gap-2">
310
- <Play className="w-4 h-4 text-gray-400" />
311
- <span className="text-sm font-medium text-gray-400">cURL Command</span>
 
 
 
 
 
 
 
312
  </div>
313
- <button
314
- onClick={() => copyToClipboard(curlExamples[activePhase], `curl-${activePhase}`)}
315
- className="text-gray-400 hover:text-white"
316
- >
317
- {copiedCode === `curl-${activePhase}` ? (
318
- <CheckCircle className="w-4 h-4 text-green-400" />
319
- ) : (
320
- <Copy className="w-4 h-4" />
321
- )}
322
- </button>
323
- </div>
324
- <pre className="bg-gray-900 rounded-lg p-4 text-sm font-mono text-gray-300 overflow-x-auto">
325
- {curlExamples[activePhase]}
326
- </pre>
327
- </div>
328
- </div>
329
- </div>
330
- </section>
331
 
332
- {/* Configuration */}
333
- <section className="mb-12">
334
- <div className="flex items-center gap-2 mb-4">
335
- <Settings className="w-5 h-5 text-indigo-400" />
336
- <h2 className="text-xl font-semibold">Configuration</h2>
337
- </div>
338
- <div className="bg-gray-800 rounded-xl border border-gray-700">
339
- <button
340
- onClick={() => toggleSection('models')}
341
- className="w-full p-4 flex items-center justify-between hover:bg-gray-700/50"
342
- >
343
- <span className="font-medium">Supported Models</span>
344
- {expandedSections['models'] ? <ChevronUp /> : <ChevronDown />}
345
- </button>
346
- {expandedSections['models'] && (
347
- <div className="p-4 border-t border-gray-700">
348
- <div className="grid grid-cols-1 md:grid-cols-2 gap-4">
349
  <div>
350
- <h4 className="font-medium mb-2 text-indigo-400">Generator Models</h4>
351
- <ul className="space-y-2 text-sm">
352
- <li className="flex items-center gap-2">
353
- <CheckCircle className="w-4 h-4 text-green-400" />
354
- <code>Qwen/Qwen2.5-7B-Instruct</code> <span className="text-gray-500">(default)</span>
355
- </li>
356
- <li className="flex items-center gap-2">
357
- <CheckCircle className="w-4 h-4 text-green-400" />
358
- <code>Qwen/Qwen2.5-72B-Instruct</code>
359
- </li>
360
- <li className="flex items-center gap-2">
361
- <CheckCircle className="w-4 h-4 text-green-400" />
362
- <code>openai/gpt-4o-mini</code>
363
- </li>
364
- </ul>
365
  </div>
 
 
366
  <div>
367
- <h4 className="font-medium mb-2 text-purple-400">Judge Models</h4>
368
- <ul className="space-y-2 text-sm">
369
- <li className="flex items-center gap-2">
370
- <CheckCircle className="w-4 h-4 text-green-400" />
371
- <code>Qwen/Qwen2.5-7B-Instruct</code> <span className="text-gray-500">(default)</span>
372
- </li>
373
- </ul>
 
 
 
374
  </div>
375
  </div>
376
  </div>
377
- )}
 
 
 
 
 
 
 
 
 
 
 
 
 
378
 
379
- <button
380
- onClick={() => toggleSection('providers')}
381
- className="w-full p-4 flex items-center justify-between hover:bg-gray-700/50 border-t border-gray-700"
382
- >
383
- <span className="font-medium">Providers</span>
384
- {expandedSections['providers'] ? <ChevronUp /> : <ChevronDown />}
385
- </button>
386
- {expandedSections['providers'] && (
387
- <div className="p-4 border-t border-gray-700">
388
- <ul className="space-y-2 text-sm">
389
- <li className="flex items-center gap-2">
390
- <CheckCircle className="w-4 h-4 text-green-400" />
391
- <code>together</code> <span className="text-gray-500">(default, recommended)</span>
392
- </li>
393
- <li className="flex items-center gap-2">
394
- <CheckCircle className="w-4 h-4 text-green-400" />
395
- <code>cerebras</code>
396
- </li>
397
- <li className="flex items-center gap-2">
398
- <CheckCircle className="w-4 h-4 text-green-400" />
399
- <code>hf</code> <span className="text-gray-500">(serverless)</span>
400
- </li>
401
- </ul>
402
- </div>
403
- )}
404
 
405
- <button
406
- onClick={() => toggleSection('env')}
407
- className="w-full p-4 flex items-center justify-between hover:bg-gray-700/50 border-t border-gray-700"
408
- >
409
- <span className="font-medium">Environment Variables</span>
410
- {expandedSections['env'] ? <ChevronUp /> : <ChevronDown />}
411
- </button>
412
- {expandedSections['env'] && (
413
- <div className="p-4 border-t border-gray-700">
414
- <pre className="bg-gray-900 rounded-lg p-4 text-sm font-mono text-gray-300">
415
- {`# Required for all spaces
416
- HF_TOKEN=your_huggingface_token_here
 
 
 
 
417
 
418
- # Optional
419
- MAX_RECORDS=1000
420
- DEFAULT_MODEL=Qwen/Qwen2.5-7B-Instruct
421
- DEFAULT_PROVIDER=together`}
422
- </pre>
 
 
 
 
 
423
  </div>
424
- )}
 
 
 
425
  </div>
426
- </section>
427
 
428
- {/* Quick Start */}
429
- <section className="mb-12">
430
- <div className="flex items-center gap-2 mb-4">
431
- <Zap className="w-5 h-5 text-indigo-400" />
432
- <h2 className="text-xl font-semibold">Quick Start</h2>
 
 
 
 
 
 
 
 
 
 
 
 
433
  </div>
434
- <div className="bg-gray-800 rounded-xl p-6 border border-gray-700">
435
- <ol className="space-y-4">
436
- <li className="flex gap-4">
437
- <span className="flex-shrink-0 w-8 h-8 rounded-full bg-indigo-500/20 text-indigo-400 flex items-center justify-center font-bold">1</span>
438
- <div>
439
- <h4 className="font-medium">Generate Seeds (Phase 0)</h4>
440
- <p className="text-sm text-gray-400 mt-1">Create initial Q&A pairs from a topic</p>
441
- </div>
442
- </li>
443
- <li className="flex gap-4">
444
- <span className="flex-shrink-0 w-8 h-8 rounded-full bg-blue-500/20 text-blue-400 flex items-center justify-center font-bold">2</span>
445
- <div>
446
- <h4 className="font-medium">Generate Synthetic Data (Phase 1)</h4>
447
- <p className="text-sm text-gray-400 mt-1">Augment with LLM Judge scoring using UltraFeedback criteria</p>
 
 
 
 
 
 
 
 
 
 
 
 
448
  </div>
449
- </li>
450
- <li className="flex gap-4">
451
- <span className="flex-shrink-0 w-8 h-8 rounded-full bg-purple-500/20 text-purple-400 flex items-center justify-center font-bold">3</span>
452
- <div>
453
- <h4 className="font-medium">Curate Dataset (Phase 2)</h4>
454
- <p className="text-sm text-gray-400 mt-1">Filter for high-quality records (score >= 7)</p>
 
 
455
  </div>
456
- </li>
457
- </ol>
458
  </div>
459
- </section>
460
-
461
- {/* Full Pipeline Script */}
462
- <section className="mb-12">
463
- <div className="flex items-center gap-2 mb-4">
464
- <GitBranch className="w-5 h-5 text-indigo-400" />
465
- <h2 className="text-xl font-semibold">Full Pipeline Script</h2>
 
 
 
 
466
  </div>
467
- <div className="bg-gray-800 rounded-xl border border-gray-700">
468
- <div className="flex items-center justify-between p-4 border-b border-gray-700">
469
- <span className="font-medium">Python Script</span>
 
470
  <button
471
- onClick={() => copyToClipboard(fullPipelineScript, 'full-script')}
472
- className="text-gray-400 hover:text-white"
473
  >
474
- {copiedCode === 'full-script' ? (
475
- <CheckCircle className="w-4 h-4 text-green-400" />
476
- ) : (
477
- <Copy className="w-4 h-4" />
478
- )}
479
  </button>
480
  </div>
481
- <pre className="p-4 text-sm font-mono text-gray-300 overflow-x-auto">
482
  {fullPipelineScript}
483
  </pre>
484
  </div>
485
- </section>
486
-
487
- {/* LLM Judge Criteria */}
488
- <section className="mb-12">
489
- <div className="flex items-center gap-2 mb-4">
490
- <AlertCircle className="w-5 h-5 text-indigo-400" />
491
- <h2 className="text-xl font-semibold">LLM Judge Scoring Criteria (UltraFeedback)</h2>
 
 
 
 
492
  </div>
493
- <div className="bg-gray-800 rounded-xl p-6 border border-gray-700">
494
- <div className="grid grid-cols-1 md:grid-cols-2 gap-6">
495
- <div>
496
- <h4 className="font-medium text-blue-400 mb-2">Instruction-Following</h4>
497
- <p className="text-sm text-gray-400">Does the answer directly address the question?</p>
498
- </div>
499
- <div>
500
- <h4 className="font-medium text-green-400 mb-2">Truthfulness</h4>
501
- <p className="text-sm text-gray-400">Is the information accurate and factually correct?</p>
502
- </div>
503
- <div>
504
- <h4 className="font-medium text-yellow-400 mb-2">Honesty</h4>
505
- <p className="text-sm text-gray-400">Does it avoid hallucination and acknowledge uncertainty?</p>
506
- </div>
507
- <div>
508
- <h4 className="font-medium text-purple-400 mb-2">Helpfulness</h4>
509
- <p className="text-sm text-gray-400">Is the answer useful for learning?</p>
510
  </div>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
511
  </div>
512
- <div className="mt-6 p-4 bg-gray-700/50 rounded-lg">
513
- <p className="text-sm text-gray-300">
514
- <strong>Score Range:</strong> 1-10 (recommended minimum: 7 for curated datasets)
515
- </p>
 
 
 
 
 
 
516
  </div>
 
517
  </div>
518
- </section>
519
-
520
- {/* Footer */}
521
- <footer className="text-center text-gray-500 text-sm py-8 border-t border-gray-800">
522
- <p>Synthetic Data Pipeline Documentation</p>
523
- <p className="mt-1">Powered by HuggingFace Spaces + HF Inference API</p>
524
- </footer>
525
- </main>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
526
  </div>
527
  )
528
  }
529
 
530
  const fullPipelineScript = `#!/usr/bin/env python3
531
- """
532
- Full Synthetic Data Pipeline Script
533
- Runs all three phases sequentially
534
- """
535
 
536
  import requests
537
  import json
538
 
539
- # Base URLs for HF Spaces
540
  PHASE0_URL = "https://mindchain-hf-space-nemo-seed-generator-huggingface-inference.hf.space"
541
  PHASE1_URL = "https://mindchain-hf-space-distilabel-generator-huggingface-inference.hf.space"
542
  PHASE2_URL = "https://mindchain-hf-space-argilla-curator-huggingface-inference.hf.space"
543
 
544
- def phase0_generate_seeds(topic, num_seeds=10):
545
  """Phase 0: Generate seed Q&A pairs"""
 
546
  response = requests.post(
547
  f"{PHASE0_URL}/generate",
548
  json={
@@ -552,10 +816,13 @@ def phase0_generate_seeds(topic, num_seeds=10):
552
  "provider": "together"
553
  }
554
  )
555
- return response.json()
 
 
556
 
557
- def phase1_generate_synthetic(seed_dataset, num_records=100):
558
  """Phase 1: Generate synthetic data with Judge scoring"""
 
559
  response = requests.post(
560
  f"{PHASE1_URL}/generate",
561
  json={
@@ -569,10 +836,14 @@ def phase1_generate_synthetic(seed_dataset, num_records=100):
569
  "min_score": 5
570
  }
571
  )
572
- return response.json()
 
 
 
573
 
574
- def phase2_curate(raw_dataset, min_score=7):
575
  """Phase 2: Curate final high-quality dataset"""
 
576
  response = requests.post(
577
  f"{PHASE2_URL}/curate",
578
  json={
@@ -582,20 +853,17 @@ def phase2_curate(raw_dataset, min_score=7):
582
  "min_score": min_score
583
  }
584
  )
585
- return response.json()
 
 
586
 
587
  if __name__ == "__main__":
588
- # Run the complete pipeline
589
- print("=== Phase 0: Generating Seeds ===")
590
  seeds = phase0_generate_seeds("Python programming basics", num_seeds=10)
591
- print(f"Generated {seeds.get('count', 0)} seeds")
592
-
593
- print("\\n=== Phase 1: Generating Synthetic Data ===")
594
  synthetic = phase1_generate_synthetic("mindchain/synthetic-seeds", num_records=100)
595
- print(f"Generated {synthetic.get('stats', {}).get('total', 0)} records")
596
-
597
- print("\\n=== Phase 2: Curating Dataset ===")
598
  curated = phase2_curate("mindchain/synthetic-distilabel-raw", min_score=7)
599
- print(f"Curated {curated.get('curated_count', 0)} high-quality records")`
 
 
600
 
601
  export default App
 
17
  ChevronDown,
18
  ChevronUp,
19
  Copy,
20
+ ExternalLink,
21
+ Sparkles,
22
+ Rocket,
23
+ Server,
24
+ Globe,
25
+ Shield,
26
+ Star,
27
+ Menu,
28
+ X
29
  } from 'lucide-react'
30
 
31
  function App() {
32
  const [activePhase, setActivePhase] = useState(0)
33
  const [expandedSections, setExpandedSections] = useState({})
34
  const [copiedCode, setCopiedCode] = useState(null)
35
+ const [mobileMenuOpen, setMobileMenuOpen] = useState(false)
36
 
37
  const toggleSection = (section) => {
38
  setExpandedSections(prev => ({
 
50
  const phases = [
51
  {
52
  id: 0,
53
+ name: 'Seed Generator',
54
+ shortName: 'Phase 0',
55
+ icon: <Zap className="w-5 h-5" />,
56
  space: 'mindchain/hf-space-nemo-seed-generator-huggingface-inference',
57
+ description: 'Generates initial seed Q&A pairs from a topic using LLM inference.',
58
+ color: 'amber',
59
+ gradient: 'from-amber-500 to-orange-600',
60
+ bgGradient: 'from-amber-500/10 to-orange-500/10',
61
  endpoint: '/generate',
62
  method: 'POST',
63
+ features: ['Topic-based generation', 'Configurable seed count', 'Quality pre-filtering'],
64
+ requestParams: [
65
+ { name: 'topic', type: 'string', desc: 'Topic for Q&A generation' },
66
+ { name: 'num_seeds', type: 'int', desc: 'Number of seeds to generate (default: 10)' },
67
+ { name: 'model', type: 'string', desc: 'Qwen/Qwen2.5-7B-Instruct' },
68
+ { name: 'provider', type: 'string', desc: 'together (recommended)' },
69
+ ],
70
  requestExample: `{
71
  "topic": "Python programming basics",
72
+ "num_seeds": 10,
73
  "model": "Qwen/Qwen2.5-7B-Instruct",
74
  "provider": "together"
75
  }`,
 
78
  "seeds": [
79
  {
80
  "instruction": "What is a variable in Python?",
81
+ "output": "A variable in Python is a named container...",
82
  "quality_score": 8
83
  }
84
  ],
85
+ "count": 10
86
  }`
87
  },
88
  {
89
  id: 1,
90
+ name: 'Distilabel Generator',
91
+ shortName: 'Phase 1',
92
+ icon: <Cpu className="w-5 h-5" />,
93
  space: 'mindchain/hf-space-distilabel-generator-huggingface-inference',
94
+ description: 'Generates synthetic data with LLM Judge scoring using UltraFeedback criteria.',
95
+ color: 'blue',
96
+ gradient: 'from-blue-500 to-cyan-500',
97
+ bgGradient: 'from-blue-500/10 to-cyan-500/10',
98
  endpoint: '/generate',
99
  method: 'POST',
100
+ features: ['UltraFeedback scoring', 'Multi-model support', 'Configurable thresholds'],
101
+ requestParams: [
102
+ { name: 'seed_dataset', type: 'string', desc: 'HF dataset ID from Phase 0' },
103
+ { name: 'num_records', type: 'int', desc: 'Records to generate (default: 100)' },
104
+ { name: 'model', type: 'string', desc: 'Generator model' },
105
+ { name: 'judge_model', type: 'string', desc: 'Judge model for scoring' },
106
+ { name: 'use_judge', type: 'bool', desc: 'Enable LLM Judge (default: true)' },
107
+ { name: 'min_score', type: 'int', desc: 'Minimum score filter (1-10)' },
108
+ ],
109
  requestExample: `{
110
  "seed_dataset": "mindchain/synthetic-seeds",
111
  "num_records": 100,
 
120
  "success": true,
121
  "records": [
122
  {
123
+ "instruction": "Explain Python loops...",
124
+ "output": "Python loops allow you to...",
125
  "quality_score": 8,
126
  "distilabel_metadata": {...}
127
  }
 
135
  },
136
  {
137
  id: 2,
138
+ name: 'Argilla Curator',
139
+ shortName: 'Phase 2',
140
+ icon: <Filter className="w-5 h-5" />,
141
  space: 'mindchain/hf-space-argilla-curator-huggingface-inference',
142
+ description: 'Final curation with quality scoring and filtering for high-quality dataset.',
143
+ color: 'purple',
144
+ gradient: 'from-purple-500 to-pink-500',
145
+ bgGradient: 'from-purple-500/10 to-pink-500/10',
146
  endpoint: '/curate',
147
  method: 'POST',
148
+ features: ['Quality filtering', 'Score distribution', 'Hub push'],
149
+ requestParams: [
150
+ { name: 'raw_dataset', type: 'string', desc: 'Raw dataset ID from Phase 1' },
151
+ { name: 'model', type: 'string', desc: 'Judge model for re-scoring' },
152
+ { name: 'provider', type: 'string', desc: 'Inference provider' },
153
+ { name: 'min_score', type: 'int', desc: 'Minimum score (default: 7)' },
154
+ { name: 'target_dataset', type: 'string', desc: 'Output dataset ID (optional)' },
155
+ ],
156
  requestExample: `{
157
  "raw_dataset": "mindchain/synthetic-distilabel-raw",
158
  "model": "Qwen/Qwen2.5-7B-Instruct",
 
166
  "total_count": 100,
167
  "filtered_count": 22,
168
  "score_distribution": {
169
+ "7": 25, "8": 30, "9": 15, "10": 8
 
 
 
170
  }
171
  }`
172
  }
173
  ]
174
 
175
+ const curlExample = (phase) => `curl -X POST https://${phase.space}.hf.space${phase.endpoint} \\
176
  -H "Content-Type: application/json" \\
177
+ -d '${phase.requestExample.replace(/\n/g, ' ')}'`
178
+
179
+ const scrollToSection = (id) => {
180
+ document.getElementById(id)?.scrollIntoView({ behavior: 'smooth' })
181
+ setMobileMenuOpen(false)
182
+ }
183
 
184
  return (
185
+ <div className="min-h-screen bg-slate-950 text-slate-100">
186
+ {/* Navigation */}
187
+ <nav className="fixed top-0 left-0 right-0 z-50 bg-slate-950/80 backdrop-blur-xl border-b border-slate-800/50">
188
+ <div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
189
+ <div className="flex items-center justify-between h-16">
190
+ <div className="flex items-center gap-3">
191
+ <div className="relative">
192
+ <div className="absolute inset-0 bg-gradient-to-r from-indigo-600 to-purple-600 rounded-xl blur-lg opacity-50"></div>
193
+ <div className="relative p-2 bg-gradient-to-r from-indigo-600 to-purple-600 rounded-xl">
194
+ <Database className="w-5 h-5" />
195
+ </div>
196
+ </div>
197
+ <div>
198
+ <h1 className="font-bold text-lg leading-tight">Synthetic Pipeline</h1>
199
+ <p className="text-xs text-slate-500">HF Spaces Documentation</p>
200
+ </div>
201
  </div>
202
+
203
+ {/* Desktop Nav */}
204
+ <div className="hidden md:flex items-center gap-6">
205
+ <button onClick={() => scrollToSection('overview')} className="text-sm text-slate-400 hover:text-white transition-colors">Overview</button>
206
+ <button onClick={() => scrollToSection('pipeline')} className="text-sm text-slate-400 hover:text-white transition-colors">Pipeline</button>
207
+ <button onClick={() => scrollToSection('api')} className="text-sm text-slate-400 hover:text-white transition-colors">API</button>
208
+ <button onClick={() => scrollToSection('quickstart')} className="text-sm text-slate-400 hover:text-white transition-colors">Quick Start</button>
209
+ <a
210
+ href="https://huggingface.co/collections/mindchain/synthetic-data-pipeline"
211
+ target="_blank"
212
+ rel="noopener noreferrer"
213
+ className="flex items-center gap-2 px-4 py-2 bg-slate-800 hover:bg-slate-700 rounded-lg text-sm font-medium transition-colors"
214
+ >
215
+ <Globe className="w-4 h-4" />
216
+ HF Collection
217
+ </a>
218
  </div>
219
+
220
+ {/* Mobile menu button */}
221
+ <button
222
+ onClick={() => setMobileMenuOpen(!mobileMenuOpen)}
223
+ className="md:hidden p-2 text-slate-400 hover:text-white"
224
+ >
225
+ {mobileMenuOpen ? <X className="w-5 h-5" /> : <Menu className="w-5 h-5" />}
226
+ </button>
227
  </div>
228
  </div>
229
+
230
+ {/* Mobile Nav */}
231
+ {mobileMenuOpen && (
232
+ <div className="md:hidden bg-slate-900 border-b border-slate-800">
233
+ <div className="px-4 py-3 space-y-2">
234
+ <button onClick={() => scrollToSection('overview')} className="block w-full text-left px-3 py-2 text-slate-400 hover:text-white hover:bg-slate-800 rounded-lg">Overview</button>
235
+ <button onClick={() => scrollToSection('pipeline')} className="block w-full text-left px-3 py-2 text-slate-400 hover:text-white hover:bg-slate-800 rounded-lg">Pipeline</button>
236
+ <button onClick={() => scrollToSection('api')} className="block w-full text-left px-3 py-2 text-slate-400 hover:text-white hover:bg-slate-800 rounded-lg">API</button>
237
+ <button onClick={() => scrollToSection('quickstart')} className="block w-full text-left px-3 py-2 text-slate-400 hover:text-white hover:bg-slate-800 rounded-lg">Quick Start</button>
238
+ </div>
239
  </div>
240
+ )}
241
+ </nav>
242
+
243
+ {/* Hero Section */}
244
+ <section className="pt-32 pb-20 px-4 relative overflow-hidden">
245
+ {/* Background Effects */}
246
+ <div className="absolute inset-0 overflow-hidden">
247
+ <div className="absolute top-1/4 -left-1/4 w-96 h-96 bg-indigo-600/20 rounded-full blur-3xl"></div>
248
+ <div className="absolute top-1/3 -right-1/4 w-96 h-96 bg-purple-600/20 rounded-full blur-3xl"></div>
249
+ <div className="absolute bottom-0 left-1/2 -translate-x-1/2 w-full h-1/2 bg-gradient-to-t from-slate-950 to-transparent"></div>
250
+ </div>
251
+
252
+ <div className="max-w-4xl mx-auto text-center relative">
253
+ <div className="inline-flex items-center gap-2 px-4 py-2 bg-slate-800/50 border border-slate-700 rounded-full text-sm text-slate-300 mb-6">
254
+ <Sparkles className="w-4 h-4 text-amber-400" />
255
+ <span>HuggingFace Spaces + HF Inference API</span>
256
+ </div>
257
+
258
+ <h1 className="text-4xl sm:text-5xl lg:text-6xl font-bold mb-6">
259
+ <span className="bg-gradient-to-r from-white via-slate-200 to-slate-400 bg-clip-text text-transparent">
260
+ Synthetic Data Pipeline
261
+ </span>
262
+ </h1>
263
+
264
+ <p className="text-lg sm:text-xl text-slate-400 max-w-2xl mx-auto mb-10">
265
+ Generate high-quality synthetic Q&A datasets with a 3-phase pipeline.
266
+ CPU-only, no GPU required.
267
+ </p>
268
+
269
+ <div className="flex flex-wrap items-center justify-center gap-4">
270
+ <button
271
+ onClick={() => scrollToSection('quickstart')}
272
+ className="btn btn-primary flex items-center gap-2"
273
+ >
274
+ <Rocket className="w-4 h-4" />
275
+ Get Started
276
+ </button>
277
+ <a
278
+ href="https://huggingface.co/collections/mindchain/synthetic-data-pipeline"
279
+ target="_blank"
280
+ rel="noopener noreferrer"
281
+ className="btn btn-secondary flex items-center gap-2"
282
+ >
283
+ <ExternalLink className="w-4 h-4" />
284
+ View Spaces
285
+ </a>
286
+ </div>
287
+ </div>
288
+ </section>
289
+
290
+ {/* Features Grid */}
291
+ <section className="py-12 px-4">
292
+ <div className="max-w-6xl mx-auto">
293
+ <div className="grid grid-cols-2 md:grid-cols-4 gap-4">
294
+ {[
295
+ { icon: <Server className="w-5 h-5" />, label: 'CPU Only', desc: 'No GPU required' },
296
+ { icon: <Zap className="w-5 h-5" />, label: 'Always On', desc: 'Docker Spaces' },
297
+ { icon: <Shield className="w-5 h-5" />, label: 'Quality Scored', desc: 'LLM Judge' },
298
+ { icon: <Globe className="w-5 h-5" />, label: 'HF Hub', desc: 'Dataset push' },
299
+ ].map((feature, i) => (
300
+ <div key={i} className="card card-hover p-4 text-center">
301
+ <div className="inline-flex p-3 bg-slate-800 rounded-xl mb-3 text-indigo-400">
302
+ {feature.icon}
303
+ </div>
304
+ <h3 className="font-semibold text-sm">{feature.label}</h3>
305
+ <p className="text-xs text-slate-500 mt-1">{feature.desc}</p>
306
  </div>
307
+ ))}
308
+ </div>
309
+ </div>
310
+ </section>
311
+
312
+ {/* Overview Section */}
313
+ <section id="overview" className="py-16 px-4">
314
+ <div className="max-w-6xl mx-auto">
315
+ <div className="flex items-center gap-3 mb-8">
316
+ <div className="p-2 bg-indigo-500/20 rounded-xl">
317
+ <BookOpen className="w-5 h-5 text-indigo-400" />
318
  </div>
319
+ <h2 className="text-2xl font-bold">Overview</h2>
320
  </div>
 
321
 
322
+ <div className="card p-6 lg:p-8">
323
+ <p className="text-slate-300 text-lg leading-relaxed mb-8">
324
+ This pipeline generates high-quality synthetic Q&A datasets using HuggingFace Spaces
325
+ and the HF Inference API. It follows a three-phase approach with LLM Judge scoring
326
+ based on UltraFeedback criteria.
327
+ </p>
328
+
329
+ <div className="grid md:grid-cols-3 gap-6">
330
+ {[
331
+ { title: 'Seed Generation', desc: 'Create initial Q&A pairs from any topic', icon: <Zap className="w-5 h-5" />, color: 'amber' },
332
+ { title: 'Distilabel Augmentation', desc: 'Generate with LLM Judge scoring', icon: <Cpu className="w-5 h-5" />, color: 'blue' },
333
+ { title: 'Quality Curation', desc: 'Filter for high-quality records only', icon: <Filter className="w-5 h-5" />, color: 'purple' },
334
+ ].map((item, i) => (
335
+ <div key={i} className="relative group">
336
+ <div className={`absolute inset-0 bg-gradient-to-r from-${item.color}-500/20 to-${item.color}-600/20 rounded-2xl blur-xl opacity-0 group-hover:opacity-100 transition-opacity`}></div>
337
+ <div className="relative bg-slate-800/50 border border-slate-700 rounded-2xl p-5 group-hover:border-slate-600 transition-colors">
338
+ <div className={`inline-flex p-2 rounded-xl bg-${item.color}-500/20 text-${item.color}-400 mb-3`}>
339
+ {item.icon}
 
 
340
  </div>
341
+ <h3 className="font-semibold mb-1">{item.title}</h3>
342
+ <p className="text-sm text-slate-400">{item.desc}</p>
343
+ </div>
344
+ </div>
 
 
345
  ))}
346
  </div>
347
  </div>
348
+ </div>
349
+ </section>
350
+
351
+ {/* Pipeline Architecture */}
352
+ <section id="pipeline" className="py-16 px-4 bg-slate-900/50">
353
+ <div className="max-w-6xl mx-auto">
354
+ <div className="flex items-center gap-3 mb-8">
355
+ <div className="p-2 bg-purple-500/20 rounded-xl">
356
+ <Layers className="w-5 h-5 text-purple-400" />
 
 
357
  </div>
358
+ <h2 className="text-2xl font-bold">Pipeline Architecture</h2>
359
+ </div>
360
 
361
+ {/* Phase Selector */}
362
+ <div className="flex flex-col lg:flex-row gap-8">
363
+ <div className="lg:w-1/3 space-y-3">
364
+ {phases.map((phase) => (
365
+ <button
366
+ key={phase.id}
367
+ onClick={() => setActivePhase(phase.id)}
368
+ className={`w-full text-left p-4 rounded-xl border-2 transition-all ${
369
+ activePhase === phase.id
370
+ ? `border-${phase.color}-500 bg-gradient-to-r ${phase.bgGradient}`
371
+ : 'border-slate-700 hover:border-slate-600 bg-slate-800/30'
372
+ }`}
373
  >
374
+ <div className="flex items-center gap-3">
375
+ <div className={`p-2 rounded-lg bg-gradient-to-r ${phase.gradient}`}>
376
+ {phase.icon}
377
+ </div>
378
+ <div>
379
+ <div className="text-xs text-slate-500 mb-0.5">{phase.shortName}</div>
380
+ <div className="font-medium">{phase.name}</div>
381
+ </div>
382
+ </div>
383
+ </button>
384
+ ))}
385
+ </div>
 
 
 
386
 
387
+ {/* Phase Details */}
388
+ <div className="lg:w-2/3">
389
+ <div className="card overflow-hidden">
390
+ <div className={`p-6 bg-gradient-to-r ${phases[activePhase].gradient}`}>
391
+ <div className="flex items-center gap-3 mb-2">
392
+ {phases[activePhase].icon}
393
+ <h3 className="text-xl font-bold">{phases[activePhase].name}</h3>
394
  </div>
395
+ <p className="text-white/80">{phases[activePhase].description}</p>
 
 
 
 
 
 
 
 
 
396
  </div>
 
 
 
 
397
 
398
+ <div className="p-6 space-y-6">
399
+ {/* Features */}
400
+ <div className="flex flex-wrap gap-2">
401
+ {phases[activePhase].features.map((f, i) => (
402
+ <span key={i} className="px-3 py-1 bg-slate-800 border border-slate-700 rounded-full text-xs">
403
+ {f}
404
+ </span>
405
+ ))}
406
  </div>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
407
 
408
+ {/* Space Link */}
 
 
409
  <div className="flex items-center gap-2">
410
+ <ExternalLink className="w-4 h-4 text-slate-500" />
411
+ <a
412
+ href={`https://huggingface.co/spaces/${phases[activePhase].space}`}
413
+ target="_blank"
414
+ rel="noopener noreferrer"
415
+ className="text-sm text-indigo-400 hover:text-indigo-300 font-mono"
416
+ >
417
+ {phases[activePhase].space}
418
+ </a>
419
  </div>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
420
 
421
+ {/* Endpoint */}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
422
  <div>
423
+ <div className="flex items-center gap-2 mb-2">
424
+ <Terminal className="w-4 h-4 text-slate-500" />
425
+ <span className="text-xs text-slate-500 uppercase tracking-wider">Endpoint</span>
426
+ <span className="px-2 py-0.5 bg-emerald-500/20 text-emerald-400 text-xs font-medium rounded">
427
+ {phases[activePhase].method}
428
+ </span>
429
+ </div>
430
+ <code className="code-block text-slate-300 text-xs sm:text-sm">
431
+ https://{phases[activePhase].space}.hf.space{phases[activePhase].endpoint}
432
+ </code>
 
 
 
 
 
433
  </div>
434
+
435
+ {/* Parameters */}
436
  <div>
437
+ <h4 className="text-xs text-slate-500 uppercase tracking-wider mb-3">Parameters</h4>
438
+ <div className="space-y-2">
439
+ {phases[activePhase].requestParams.map((param, i) => (
440
+ <div key={i} className="flex flex-wrap items-center gap-2 text-sm">
441
+ <code className="px-2 py-1 bg-slate-800 rounded text-indigo-300 font-mono">{param.name}</code>
442
+ <span className="text-slate-500 text-xs">{param.type}</span>
443
+ <span className="text-slate-400">{param.desc}</span>
444
+ </div>
445
+ ))}
446
+ </div>
447
  </div>
448
  </div>
449
  </div>
450
+ </div>
451
+ </div>
452
+ </div>
453
+ </section>
454
+
455
+ {/* API Examples Section */}
456
+ <section id="api" className="py-16 px-4">
457
+ <div className="max-w-6xl mx-auto">
458
+ <div className="flex items-center gap-3 mb-8">
459
+ <div className="p-2 bg-emerald-500/20 rounded-xl">
460
+ <Code className="w-5 h-5 text-emerald-400" />
461
+ </div>
462
+ <h2 className="text-2xl font-bold">API Examples</h2>
463
+ </div>
464
 
465
+ {/* Phase Tabs */}
466
+ <div className="flex gap-2 mb-6 overflow-x-auto pb-2">
467
+ {phases.map((phase) => (
468
+ <button
469
+ key={phase.id}
470
+ onClick={() => setActivePhase(phase.id)}
471
+ className={`flex items-center gap-2 px-4 py-2 rounded-lg font-medium text-sm whitespace-nowrap transition-colors ${
472
+ activePhase === phase.id
473
+ ? `bg-gradient-to-r ${phase.gradient} text-white`
474
+ : 'bg-slate-800 text-slate-400 hover:text-white'
475
+ }`}
476
+ >
477
+ {phase.icon}
478
+ {phase.shortName}
479
+ </button>
480
+ ))}
481
+ </div>
 
 
 
 
 
 
 
 
482
 
483
+ <div className="grid lg:grid-cols-2 gap-6">
484
+ {/* Request */}
485
+ <div className="card overflow-hidden">
486
+ <div className="flex items-center justify-between px-4 py-3 bg-slate-800/50 border-b border-slate-700">
487
+ <span className="text-sm font-medium text-slate-300">Request Body</span>
488
+ <button
489
+ onClick={() => copyToClipboard(phases[activePhase].requestExample, 'req')}
490
+ className="text-slate-400 hover:text-white transition-colors"
491
+ >
492
+ {copiedCode === 'req' ? <CheckCircle className="w-4 h-4 text-emerald-400" /> : <Copy className="w-4 h-4" />}
493
+ </button>
494
+ </div>
495
+ <pre className="p-4 text-sm font-mono text-slate-300 overflow-x-auto scrollbar-thin">
496
+ {phases[activePhase].requestExample}
497
+ </pre>
498
+ </div>
499
 
500
+ {/* Response */}
501
+ <div className="card overflow-hidden">
502
+ <div className="flex items-center justify-between px-4 py-3 bg-slate-800/50 border-b border-slate-700">
503
+ <span className="text-sm font-medium text-slate-300">Response</span>
504
+ <button
505
+ onClick={() => copyToClipboard(phases[activePhase].responseExample, 'res')}
506
+ className="text-slate-400 hover:text-white transition-colors"
507
+ >
508
+ {copiedCode === 'res' ? <CheckCircle className="w-4 h-4 text-emerald-400" /> : <Copy className="w-4 h-4" />}
509
+ </button>
510
  </div>
511
+ <pre className="p-4 text-sm font-mono text-slate-300 overflow-x-auto scrollbar-thin">
512
+ {phases[activePhase].responseExample}
513
+ </pre>
514
+ </div>
515
  </div>
 
516
 
517
+ {/* cURL */}
518
+ <div className="card overflow-hidden mt-6">
519
+ <div className="flex items-center justify-between px-4 py-3 bg-slate-800/50 border-b border-slate-700">
520
+ <div className="flex items-center gap-2">
521
+ <Terminal className="w-4 h-4 text-slate-500" />
522
+ <span className="text-sm font-medium text-slate-300">cURL Command</span>
523
+ </div>
524
+ <button
525
+ onClick={() => copyToClipboard(curlExample(phases[activePhase]), 'curl')}
526
+ className="text-slate-400 hover:text-white transition-colors"
527
+ >
528
+ {copiedCode === 'curl' ? <CheckCircle className="w-4 h-4 text-emerald-400" /> : <Copy className="w-4 h-4" />}
529
+ </button>
530
+ </div>
531
+ <pre className="p-4 text-sm font-mono text-slate-300 overflow-x-auto scrollbar-thin">
532
+ {curlExample(phases[activePhase])}
533
+ </pre>
534
  </div>
535
+ </div>
536
+ </section>
537
+
538
+ {/* Quick Start */}
539
+ <section id="quickstart" className="py-16 px-4 bg-slate-900/50">
540
+ <div className="max-w-6xl mx-auto">
541
+ <div className="flex items-center gap-3 mb-8">
542
+ <div className="p-2 bg-amber-500/20 rounded-xl">
543
+ <Rocket className="w-5 h-5 text-amber-400" />
544
+ </div>
545
+ <h2 className="text-2xl font-bold">Quick Start</h2>
546
+ </div>
547
+
548
+ <div className="grid lg:grid-cols-3 gap-6">
549
+ {[
550
+ { step: 1, title: 'Generate Seeds', desc: 'Create initial Q&A pairs from your topic', color: 'amber', phase: 'Phase 0' },
551
+ { step: 2, title: 'Generate Data', desc: 'Augment with LLM Judge quality scoring', color: 'blue', phase: 'Phase 1' },
552
+ { step: 3, title: 'Curate Dataset', desc: 'Filter high-quality records (score >= 7)', color: 'purple', phase: 'Phase 2' },
553
+ ].map((item) => (
554
+ <div key={item.step} className="card card-hover p-6 relative">
555
+ <div className={`absolute -top-3 -left-3 w-8 h-8 rounded-full bg-gradient-to-r ${
556
+ item.color === 'amber' ? 'from-amber-500 to-orange-500' :
557
+ item.color === 'blue' ? 'from-blue-500 to-cyan-500' :
558
+ 'from-purple-500 to-pink-500'
559
+ } flex items-center justify-center font-bold text-sm`}>
560
+ {item.step}
561
  </div>
562
+ <div className="pt-2">
563
+ <span className={`text-xs font-medium ${
564
+ item.color === 'amber' ? 'text-amber-400' :
565
+ item.color === 'blue' ? 'text-blue-400' :
566
+ 'text-purple-400'
567
+ }`}>{item.phase}</span>
568
+ <h3 className="font-semibold text-lg mt-1">{item.title}</h3>
569
+ <p className="text-sm text-slate-400 mt-2">{item.desc}</p>
570
  </div>
571
+ </div>
572
+ ))}
573
  </div>
574
+ </div>
575
+ </section>
576
+
577
+ {/* Full Script */}
578
+ <section className="py-16 px-4">
579
+ <div className="max-w-6xl mx-auto">
580
+ <div className="flex items-center gap-3 mb-8">
581
+ <div className="p-2 bg-indigo-500/20 rounded-xl">
582
+ <GitBranch className="w-5 h-5 text-indigo-400" />
583
+ </div>
584
+ <h2 className="text-2xl font-bold">Full Pipeline Script</h2>
585
  </div>
586
+
587
+ <div className="card overflow-hidden">
588
+ <div className="flex items-center justify-between px-4 py-3 bg-slate-800/50 border-b border-slate-700">
589
+ <span className="text-sm font-medium text-slate-300">run_pipeline.py</span>
590
  <button
591
+ onClick={() => copyToClipboard(fullPipelineScript, 'script')}
592
+ className="text-slate-400 hover:text-white transition-colors"
593
  >
594
+ {copiedCode === 'script' ? <CheckCircle className="w-4 h-4 text-emerald-400" /> : <Copy className="w-4 h-4" />}
 
 
 
 
595
  </button>
596
  </div>
597
+ <pre className="p-4 text-sm font-mono text-slate-300 overflow-x-auto scrollbar-thin max-h-96">
598
  {fullPipelineScript}
599
  </pre>
600
  </div>
601
+ </div>
602
+ </section>
603
+
604
+ {/* LLM Judge Criteria */}
605
+ <section className="py-16 px-4 bg-slate-900/50">
606
+ <div className="max-w-6xl mx-auto">
607
+ <div className="flex items-center gap-3 mb-8">
608
+ <div className="p-2 bg-pink-500/20 rounded-xl">
609
+ <Star className="w-5 h-5 text-pink-400" />
610
+ </div>
611
+ <h2 className="text-2xl font-bold">LLM Judge Criteria (UltraFeedback)</h2>
612
  </div>
613
+
614
+ <div className="grid sm:grid-cols-2 lg:grid-cols-4 gap-4">
615
+ {[
616
+ { title: 'Instruction-Following', desc: 'Does the answer directly address the question?', color: 'blue' },
617
+ { title: 'Truthfulness', desc: 'Is the information accurate and factually correct?', color: 'green' },
618
+ { title: 'Honesty', desc: 'Does it avoid hallucination and acknowledge uncertainty?', color: 'yellow' },
619
+ { title: 'Helpfulness', desc: 'Is the answer useful for learning?', color: 'purple' },
620
+ ].map((item, i) => (
621
+ <div key={i} className="card card-hover p-5">
622
+ <div className={`w-10 h-1 rounded-full mb-4 bg-${item.color}-500`}></div>
623
+ <h3 className="font-semibold mb-2">{item.title}</h3>
624
+ <p className="text-sm text-slate-400">{item.desc}</p>
 
 
 
 
 
625
  </div>
626
+ ))}
627
+ </div>
628
+
629
+ <div className="card mt-8 p-6 flex flex-col sm:flex-row items-center justify-between gap-4">
630
+ <div>
631
+ <h3 className="font-semibold">Score Range</h3>
632
+ <p className="text-sm text-slate-400">1-10 scale, recommended minimum: 7 for curated datasets</p>
633
+ </div>
634
+ <div className="flex gap-2">
635
+ {[1, 2, 3, 4, 5, 6, 7, 8, 9, 10].map((n) => (
636
+ <div
637
+ key={n}
638
+ className={`w-8 h-8 rounded-lg flex items-center justify-center text-xs font-bold ${
639
+ n >= 7 ? 'bg-emerald-500/20 text-emerald-400 border border-emerald-500/30' : 'bg-slate-800 text-slate-500'
640
+ }`}
641
+ >
642
+ {n}
643
+ </div>
644
+ ))}
645
  </div>
646
+ </div>
647
+ </div>
648
+ </section>
649
+
650
+ {/* Configuration */}
651
+ <section className="py-16 px-4">
652
+ <div className="max-w-6xl mx-auto">
653
+ <div className="flex items-center gap-3 mb-8">
654
+ <div className="p-2 bg-slate-500/20 rounded-xl">
655
+ <Settings className="w-5 h-5 text-slate-400" />
656
  </div>
657
+ <h2 className="text-2xl font-bold">Configuration</h2>
658
  </div>
659
+
660
+ <div className="card divide-y divide-slate-800">
661
+ {/* Models */}
662
+ <div>
663
+ <button
664
+ onClick={() => toggleSection('models')}
665
+ className="w-full flex items-center justify-between p-5 hover:bg-slate-800/30 transition-colors"
666
+ >
667
+ <span className="font-medium">Supported Models</span>
668
+ {expandedSections['models'] ? <ChevronUp className="w-5 h-5 text-slate-400" /> : <ChevronDown className="w-5 h-5 text-slate-400" />}
669
+ </button>
670
+ {expandedSections['models'] && (
671
+ <div className="px-5 pb-5">
672
+ <div className="grid sm:grid-cols-2 gap-6">
673
+ <div>
674
+ <h4 className="text-sm font-medium text-indigo-400 mb-3">Generator Models</h4>
675
+ <div className="space-y-2">
676
+ {[
677
+ { name: 'Qwen/Qwen2.5-7B-Instruct', default: true },
678
+ { name: 'Qwen/Qwen2.5-72B-Instruct', default: false },
679
+ { name: 'openai/gpt-4o-mini', default: false },
680
+ ].map((m, i) => (
681
+ <div key={i} className="flex items-center gap-2 text-sm">
682
+ <CheckCircle className="w-4 h-4 text-emerald-400 flex-shrink-0" />
683
+ <code className="font-mono text-slate-300">{m.name}</code>
684
+ {m.default && <span className="text-xs text-slate-500">(default)</span>}
685
+ </div>
686
+ ))}
687
+ </div>
688
+ </div>
689
+ <div>
690
+ <h4 className="text-sm font-medium text-purple-400 mb-3">Judge Models</h4>
691
+ <div className="space-y-2">
692
+ <div className="flex items-center gap-2 text-sm">
693
+ <CheckCircle className="w-4 h-4 text-emerald-400" />
694
+ <code className="font-mono text-slate-300">Qwen/Qwen2.5-7B-Instruct</code>
695
+ <span className="text-xs text-slate-500">(default)</span>
696
+ </div>
697
+ </div>
698
+ </div>
699
+ </div>
700
+ </div>
701
+ )}
702
+ </div>
703
+
704
+ {/* Providers */}
705
+ <div>
706
+ <button
707
+ onClick={() => toggleSection('providers')}
708
+ className="w-full flex items-center justify-between p-5 hover:bg-slate-800/30 transition-colors"
709
+ >
710
+ <span className="font-medium">Providers</span>
711
+ {expandedSections['providers'] ? <ChevronUp className="w-5 h-5 text-slate-400" /> : <ChevronDown className="w-5 h-5 text-slate-400" />}
712
+ </button>
713
+ {expandedSections['providers'] && (
714
+ <div className="px-5 pb-5">
715
+ <div className="space-y-2">
716
+ {[
717
+ { name: 'together', desc: '(default, recommended)' },
718
+ { name: 'cerebras', desc: '' },
719
+ { name: 'hf', desc: '(serverless)' },
720
+ ].map((p, i) => (
721
+ <div key={i} className="flex items-center gap-2 text-sm">
722
+ <CheckCircle className="w-4 h-4 text-emerald-400" />
723
+ <code className="font-mono text-slate-300">{p.name}</code>
724
+ {p.desc && <span className="text-xs text-slate-500">{p.desc}</span>}
725
+ </div>
726
+ ))}
727
+ </div>
728
+ </div>
729
+ )}
730
+ </div>
731
+
732
+ {/* Environment */}
733
+ <div>
734
+ <button
735
+ onClick={() => toggleSection('env')}
736
+ className="w-full flex items-center justify-between p-5 hover:bg-slate-800/30 transition-colors"
737
+ >
738
+ <span className="font-medium">Environment Variables</span>
739
+ {expandedSections['env'] ? <ChevronUp className="w-5 h-5 text-slate-400" /> : <ChevronDown className="w-5 h-5 text-slate-400" />}
740
+ </button>
741
+ {expandedSections['env'] && (
742
+ <div className="px-5 pb-5">
743
+ <pre className="code-block text-sm">
744
+ {`# Required for all spaces
745
+ HF_TOKEN=your_huggingface_token_here
746
+
747
+ # Optional
748
+ MAX_RECORDS=1000
749
+ DEFAULT_MODEL=Qwen/Qwen2.5-7B-Instruct
750
+ DEFAULT_PROVIDER=together`}
751
+ </pre>
752
+ </div>
753
+ )}
754
+ </div>
755
+ </div>
756
+ </div>
757
+ </section>
758
+
759
+ {/* Footer */}
760
+ <footer className="py-12 px-4 border-t border-slate-800">
761
+ <div className="max-w-6xl mx-auto text-center">
762
+ <div className="flex items-center justify-center gap-2 mb-4">
763
+ <div className="p-2 bg-gradient-to-r from-indigo-600 to-purple-600 rounded-xl">
764
+ <Database className="w-4 h-4" />
765
+ </div>
766
+ <span className="font-semibold">Synthetic Data Pipeline</span>
767
+ </div>
768
+ <p className="text-sm text-slate-500 mb-4">
769
+ Powered by HuggingFace Spaces + HF Inference API
770
+ </p>
771
+ <div className="flex items-center justify-center gap-4">
772
+ <a
773
+ href="https://huggingface.co/mindchain"
774
+ target="_blank"
775
+ rel="noopener noreferrer"
776
+ className="text-sm text-slate-400 hover:text-white transition-colors"
777
+ >
778
+ HuggingFace
779
+ </a>
780
+ <span className="text-slate-700">|</span>
781
+ <a
782
+ href="https://huggingface.co/collections/mindchain/synthetic-data-pipeline"
783
+ target="_blank"
784
+ rel="noopener noreferrer"
785
+ className="text-sm text-slate-400 hover:text-white transition-colors"
786
+ >
787
+ Collection
788
+ </a>
789
+ </div>
790
+ </div>
791
+ </footer>
792
  </div>
793
  )
794
  }
795
 
796
  const fullPipelineScript = `#!/usr/bin/env python3
797
+ """Full Synthetic Data Pipeline - Run all three phases"""
 
 
 
798
 
799
  import requests
800
  import json
801
 
802
+ # HF Spaces URLs
803
  PHASE0_URL = "https://mindchain-hf-space-nemo-seed-generator-huggingface-inference.hf.space"
804
  PHASE1_URL = "https://mindchain-hf-space-distilabel-generator-huggingface-inference.hf.space"
805
  PHASE2_URL = "https://mindchain-hf-space-argilla-curator-huggingface-inference.hf.space"
806
 
807
+ def phase0_generate_seeds(topic: str, num_seeds: int = 10):
808
  """Phase 0: Generate seed Q&A pairs"""
809
+ print(f"\\n=== Phase 0: Generating {num_seeds} seeds for '{topic}' ===")
810
  response = requests.post(
811
  f"{PHASE0_URL}/generate",
812
  json={
 
816
  "provider": "together"
817
  }
818
  )
819
+ result = response.json()
820
+ print(f"Generated {result.get('count', 0)} seeds")
821
+ return result
822
 
823
+ def phase1_generate_synthetic(seed_dataset: str, num_records: int = 100):
824
  """Phase 1: Generate synthetic data with Judge scoring"""
825
+ print(f"\\n=== Phase 1: Generating {num_records} records ===")
826
  response = requests.post(
827
  f"{PHASE1_URL}/generate",
828
  json={
 
836
  "min_score": 5
837
  }
838
  )
839
+ result = response.json()
840
+ stats = result.get('stats', {})
841
+ print(f"Generated {stats.get('total', 0)} records, avg score: {stats.get('avg_score', 0):.1f}")
842
+ return result
843
 
844
+ def phase2_curate(raw_dataset: str, min_score: int = 7):
845
  """Phase 2: Curate final high-quality dataset"""
846
+ print(f"\\n=== Phase 2: Curating with min_score={min_score} ===")
847
  response = requests.post(
848
  f"{PHASE2_URL}/curate",
849
  json={
 
853
  "min_score": min_score
854
  }
855
  )
856
+ result = response.json()
857
+ print(f"Curated {result.get('curated_count', 0)}/{result.get('total_count', 0)} records")
858
+ return result
859
 
860
  if __name__ == "__main__":
861
+ # Run complete pipeline
 
862
  seeds = phase0_generate_seeds("Python programming basics", num_seeds=10)
 
 
 
863
  synthetic = phase1_generate_synthetic("mindchain/synthetic-seeds", num_records=100)
 
 
 
864
  curated = phase2_curate("mindchain/synthetic-distilabel-raw", min_score=7)
865
+
866
+ print("\\n=== Pipeline Complete! ===")
867
+ print(f"Final curated dataset: {curated.get('curated_count', 0)} high-quality records")`
868
 
869
  export default App