Skip to content

Commit 18ec6ef

Browse files
yamalightyamalight
committed
Add new empty source, add example for it
Add example showing how to use code to dynamically fetch input data Addresses #15
1 parent 451b90f commit 18ec6ef

File tree

6 files changed

+174
-3
lines changed

6 files changed

+174
-3
lines changed

‎app/components/pipeline/nodes/source/EmptySource.tsx

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
import { Doc } from 'litlytics';
2+
import { useEffect } from 'react';
3+
4+
export function EmptySourceRender({
5+
setDocs,
6+
}: {
7+
docs: Doc[];
8+
setDocs: (newDocs: Doc[]) => void;
9+
}) {
10+
useEffect(() => {
11+
const newDoc = {
12+
id: 'textdoc',
13+
name: 'Default document',
14+
content: 'empty',
15+
test: true,
16+
processingResults: [],
17+
};
18+
setDocs([newDoc]);
19+
}, [setDocs]);
20+
21+
return (
22+
<div className="flex flex-col w-full h-full overflow-auto">
23+
<div className="flex flex-col">Empty source</div>
24+
</div>
25+
);
26+
}

‎app/components/pipeline/nodes/source/SourceNode.tsx

+18-3
Original file line numberDiff line numberDiff line change
@@ -91,9 +91,18 @@ export function SourceNode() {
9191
<NodeFrame
9292
hasConnector={litlytics.pipeline.steps.length > 0 ? true : 'auto'}
9393
currentStep={source}
94-
size={source.expanded ? 'lg' : 'collapsed'}
94+
size={
95+
// always collapse empty
96+
sourceType === 'empty'
97+
? 'collapsed'
98+
: source.expanded
99+
? 'lg'
100+
: 'collapsed'
101+
}
95102
>
96-
<NodeHeader collapsed={!source.expanded}>
103+
<NodeHeader
104+
collapsed={sourceType === 'empty' ? true : !source.expanded}
105+
>
97106
<div className="flex flex-1 gap-2 items-center">
98107
{litlytics.pipelineStatus.status === 'sourcing' ? (
99108
<Spinner className="w-4 h-4" />
@@ -103,7 +112,13 @@ export function SourceNode() {
103112
className="!p-0"
104113
onClick={() => updateNodeByKey(!source.expanded, 'expanded')}
105114
>
106-
{source.expanded ? <ChevronDownIcon /> : <ChevronRightIcon />}
115+
{sourceType === 'empty' ? (
116+
<ChevronRightIcon />
117+
) : source.expanded ? (
118+
<ChevronDownIcon />
119+
) : (
120+
<ChevronRightIcon />
121+
)}
107122
</Button>
108123
)}
109124
<RectangleStackIcon className="w-4 h-4" /> Source

‎app/components/pipeline/nodes/source/providers.ts

+2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { DocsListSourceRender } from './docsList/DocsListRender';
2+
import { EmptySourceRender } from './EmptySource';
23
import { FileSourceRender } from './file/FileSource';
34
import { TextSourceRender } from './TextSource';
45
import { SourceTypes } from './types';
@@ -9,4 +10,5 @@ export const sourceRenders = {
910
[SourceTypes.TEXT]: TextSourceRender,
1011
[SourceTypes.FILE]: FileSourceRender,
1112
[SourceTypes.URL]: URLSourceRender,
13+
[SourceTypes.EMPTY]: EmptySourceRender,
1214
};

‎app/components/pipeline/nodes/source/types.ts

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ export class SourceTypes {
33
static DOCS = 'docs' as const;
44
static FILE = 'file' as const;
55
static URL = 'url' as const;
6+
static EMPTY = 'empty' as const;
67
}
78

89
type SourceTypeList = (typeof SourceTypes)[keyof typeof SourceTypes];

‎examples/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@ You can find following example pipeline for LitLytics in this folder:
55
- [Converting text into structured formats like JSON-LD](./jsonld.json)
66
- [Writing Instagram posts with proof-reading for clarity and tone](./writing.json)
77
- [Writing Etsy product description with copyright checks and SEO improvements](./etsy.json)
8+
- [Using empty source and first code step to fetch data from APIs](./code-source.json)

‎examples/code-source.json

+126
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
{
2+
"name": "code-source",
3+
"pipelinePlan": "",
4+
"pipelineDescription": "Fetch github user profile and verbalize it",
5+
"source": {
6+
"id": "source_0",
7+
"name": "Source",
8+
"description": "Primary source",
9+
"type": "source",
10+
"sourceType": "text",
11+
"docs": [
12+
{
13+
"id": "textdoc",
14+
"name": "Default document",
15+
"content": "empty",
16+
"test": true,
17+
"processingResults": [
18+
{
19+
"stepId": "step_0",
20+
"result": "Username: yamalight\nName: Tim Ermilov\nBio: Hi, I'm Tim. I do javascript, webdev and knowledge graphs.\n\n\nRepositories:\n- actions-mongo-test: Basic test of importing dump into mongodb in github actions\n- feedly-colorful-list-view: Tiny userscript that colorizes Feedly news entries by source.\n- gaming-knowledge-graph: Building a gaming knowledge graph\n- gatsby-mdx-bug-demo: Simple demo for MDX graphql issue\n- generator-powder: Powder.js Yeoman generator\n- graffiti: Minimalistic GraphQL framework\n- gsmCharacteristics: gsmCharacteristics parsing and transforming project\n- HumblePlayer: HumblePlayer - music player for your HumbleBundle soundtracks\n- libcodezen: Adobe AIR, flex, as3 classes and components library\n- litlytics: 🔥 LitLytics - an affordable, simple analytics platform that leverages LLMs to automate data analysis\n- microcore: Core library for simple creation of pipelinening microservices in Node.js with RabbitMQ\n- microwork: Microwork - simple creation of distributed scalable microservices in node.js with RabbitMQ\n- MobilePOIStudentProject: null\n- node-docker-pkg-demo: Demo of using Docker multi-stage builds with Zeit pkg to build small node-based images\n- outstated: Simple hooks-based state management for React\n- particula: Zero-config Express.js Framework\n- particula-core-fastify: Fastify.js core for Particula\n- particula-plugin-next: Particula plugin that adds Next.js integration\n- postal.observe: A postal.js plugin that provides a way to get a subscription as Rx.Observable\n- presentation-asyncjs: Presentation on working with asynchronous javascript (promises, async/await, functional programming, reactive programming)\n- presentation-rdf-intro: Intoduction to RDF and the Web of Data (presentation)\n- reddmix: Remix-based Reddit client\n- rethinkdb-pubsub: Implementation of message queueing on top of RethinkDB changefeeds as a library",
21+
"timingMs": 713
22+
},
23+
{
24+
"stepId": "step_1",
25+
"result": "**GitHub Profile Summary for Tim Ermilov (yamalight)**\n\nTim Ermilov, known by the username yamalight, is a proficient developer specializing in JavaScript, web development, and knowledge graphs. His GitHub profile showcases a diverse array of projects, reflecting his expertise in both front-end and back-end technologies.\n\nKey projects include:\n- **Gaming Knowledge Graph**: A project focused on building a comprehensive knowledge graph for gaming.\n- **LitLytics**: An innovative analytics platform that utilizes large language models (LLMs) for automated data analysis.\n- **Graffiti**: A minimalistic GraphQL framework, demonstrating his skills in modern web technologies.\n- **Particula**: A zero-config Express.js framework, along with its Fastify.js core and Next.js integration, highlighting his work in creating streamlined development tools.\n\nTim has also contributed to various utility projects, such as a music player for HumbleBundle soundtracks and a simple state management solution for React. His repositories indicate a strong focus on microservices, with multiple projects utilizing Node.js and RabbitMQ for scalable applications.\n\nOverall, Tim's GitHub profile reflects a commitment to open-source development and a passion for creating tools that enhance web development and data management.",
26+
"usage": {
27+
"completionTokens": 251,
28+
"promptTokens": 607,
29+
"totalTokens": 858
30+
},
31+
"timingMs": 3281
32+
}
33+
]
34+
}
35+
],
36+
"connectsTo": [
37+
"step_0"
38+
],
39+
"config": {
40+
"type": "empty"
41+
},
42+
"expanded": false
43+
},
44+
"steps": [
45+
{
46+
"id": "step_0",
47+
"name": "Fetch github profile",
48+
"description": "Fetch public github profile for given username",
49+
"type": "code",
50+
"input": "doc",
51+
"code": "export default async function fetchGithubProfileAndRepos(_doc) {\n const doc = 'yamalight';\n const profileUrl = `https://api.github.com/users/${doc}`;\n \n try {\n const profileResponse = await fetch(profileUrl);\n if (!profileResponse.ok) {\n return undefined;\n }\n const profileData = await profileResponse.json();\n \n let result = `Username: ${profileData.login}\nName: ${profileData.name}\nBio: ${profileData.bio}\n`;\n\n const reposUrl = `https://api.github.com/users/${doc}/repos`;\n const reposResponse = await fetch(reposUrl);\n if (!reposResponse.ok) {\n return undefined;\n }\n const reposData = await reposResponse.json();\n \n const reposList = reposData.filter(repo => !repo.fork).map(repo => `- ${repo.name}: ${repo.description}`).join('\\n');\n result += `\n\nRepositories:\n${reposList}`;\n\n return result;\n } catch (error) {\n return undefined;\n }\n}",
52+
"codeExplanation": "This function retrieves information from GitHub about a specific user and their repositories. It starts by taking the user's username and fetching their profile details, such as their username, name, and bio. If this information is successfully obtained, it then looks for the user's repositories and compiles a list of them. Finally, it combines all this information into a clear format and returns it. If there are any issues during this process, it simply indicates that the information could not be retrieved.",
53+
"connectsTo": [
54+
"step_1"
55+
],
56+
"expanded": true
57+
},
58+
{
59+
"id": "step_1",
60+
"name": "Summarize github profile",
61+
"description": "Look at given github profile and user repos and write a short summary of the user profile.",
62+
"type": "llm",
63+
"input": "result",
64+
"prompt": "You are tasked with summarizing a GitHub profile based on the provided information. Please follow these steps:\n\n1. **Analyze the Profile**: Carefully examine the user's GitHub profile, including their repositories, contributions, and any other relevant information.\n2. **Identify Key Aspects**: Look for important details such as the user's primary programming languages, types of projects they work on, and any notable contributions to open-source projects.\n3. **Summarize**: Write a concise summary that captures the essence of the user's GitHub profile. Highlight their skills, areas of expertise, and any significant achievements or projects.\n4. **Clarity and Brevity**: Ensure that the summary is clear and to the point, avoiding unnecessary jargon or overly complex language.\n\nPlease provide a short summary of the user profile based on the information given.",
65+
"connectsTo": [
66+
"litlytics_output"
67+
],
68+
"expanded": true
69+
}
70+
],
71+
"results": [
72+
{
73+
"stepId": "step_1",
74+
"result": "**GitHub Profile Summary for Tim Ermilov (yamalight)**\n\nTim Ermilov, known by the username yamalight, is a proficient developer specializing in JavaScript, web development, and knowledge graphs. His GitHub profile showcases a diverse array of projects, reflecting his expertise in both front-end and back-end technologies.\n\nKey projects include:\n- **Gaming Knowledge Graph**: A project focused on building a comprehensive knowledge graph for gaming.\n- **LitLytics**: An innovative analytics platform that utilizes large language models (LLMs) for automated data analysis.\n- **Graffiti**: A minimalistic GraphQL framework, demonstrating his capability in modern web technologies.\n- **Particula**: A zero-config Express.js framework, along with its Fastify.js core and Next.js integration, highlighting his work in creating streamlined development tools.\n\nTim has also contributed to various utility projects, such as a music player for HumbleBundle soundtracks and a simple state management solution for React. His repositories indicate a strong focus on microservices architecture, with multiple projects utilizing Node.js and RabbitMQ.\n\nOverall, Tim's GitHub profile reflects a commitment to open-source development and a passion for creating tools that enhance web development and data management.",
75+
"usage": {
76+
"completionTokens": 249,
77+
"promptTokens": 607,
78+
"totalTokens": 856
79+
},
80+
"timingMs": 3012,
81+
"doc": {
82+
"id": "textdoc",
83+
"name": "Default document",
84+
"content": "empty",
85+
"test": true,
86+
"processingResults": [
87+
{
88+
"stepId": "step_1",
89+
"result": "**GitHub Profile Summary for Tim Ermilov (yamalight)**\n\nTim Ermilov, known by the username yamalight, is a proficient developer specializing in JavaScript, web development, and knowledge graphs. His GitHub profile showcases a diverse array of projects, reflecting his expertise in both front-end and back-end technologies.\n\nKey projects include:\n- **Gaming Knowledge Graph**: A project focused on building a comprehensive knowledge graph for gaming.\n- **LitLytics**: An innovative analytics platform that utilizes large language models (LLMs) for automated data analysis.\n- **Graffiti**: A minimalistic GraphQL framework, demonstrating his capability in modern web technologies.\n- **Particula**: A zero-config Express.js framework, along with its Fastify.js core and Next.js integration, highlighting his work in creating streamlined development tools.\n\nTim has also contributed to various utility projects, such as a music player for HumbleBundle soundtracks and a simple state management solution for React. His repositories indicate a strong focus on microservices architecture, with multiple projects utilizing Node.js and RabbitMQ.\n\nOverall, Tim's GitHub profile reflects a commitment to open-source development and a passion for creating tools that enhance web development and data management.",
90+
"usage": {
91+
"completionTokens": 249,
92+
"promptTokens": 607,
93+
"totalTokens": 856
94+
},
95+
"timingMs": 3012
96+
}
97+
]
98+
}
99+
}
100+
],
101+
"resultDocs": [
102+
{
103+
"id": "textdoc",
104+
"name": "Default document",
105+
"content": "empty",
106+
"test": true,
107+
"processingResults": [
108+
{
109+
"stepId": "step_0",
110+
"result": "Username: yamalight\nName: Tim Ermilov\nBio: Hi, I'm Tim. I do javascript, webdev and knowledge graphs.\n\n\nRepositories:\n- actions-mongo-test: Basic test of importing dump into mongodb in github actions\n- feedly-colorful-list-view: Tiny userscript that colorizes Feedly news entries by source.\n- gaming-knowledge-graph: Building a gaming knowledge graph\n- gatsby-mdx-bug-demo: Simple demo for MDX graphql issue\n- generator-powder: Powder.js Yeoman generator\n- graffiti: Minimalistic GraphQL framework\n- gsmCharacteristics: gsmCharacteristics parsing and transforming project\n- HumblePlayer: HumblePlayer - music player for your HumbleBundle soundtracks\n- libcodezen: Adobe AIR, flex, as3 classes and components library\n- litlytics: 🔥 LitLytics - an affordable, simple analytics platform that leverages LLMs to automate data analysis\n- microcore: Core library for simple creation of pipelinening microservices in Node.js with RabbitMQ\n- microwork: Microwork - simple creation of distributed scalable microservices in node.js with RabbitMQ\n- MobilePOIStudentProject: null\n- node-docker-pkg-demo: Demo of using Docker multi-stage builds with Zeit pkg to build small node-based images\n- outstated: Simple hooks-based state management for React\n- particula: Zero-config Express.js Framework\n- particula-core-fastify: Fastify.js core for Particula\n- particula-plugin-next: Particula plugin that adds Next.js integration\n- postal.observe: A postal.js plugin that provides a way to get a subscription as Rx.Observable\n- presentation-asyncjs: Presentation on working with asynchronous javascript (promises, async/await, functional programming, reactive programming)\n- presentation-rdf-intro: Intoduction to RDF and the Web of Data (presentation)\n- reddmix: Remix-based Reddit client\n- rethinkdb-pubsub: Implementation of message queueing on top of RethinkDB changefeeds as a library",
111+
"timingMs": 713
112+
},
113+
{
114+
"stepId": "step_1",
115+
"result": "**GitHub Profile Summary for Tim Ermilov (yamalight)**\n\nTim Ermilov, known by the username yamalight, is a proficient developer specializing in JavaScript, web development, and knowledge graphs. His GitHub profile showcases a diverse array of projects, reflecting his expertise in both front-end and back-end technologies.\n\nKey projects include:\n- **Gaming Knowledge Graph**: A project focused on building a comprehensive knowledge graph for gaming.\n- **LitLytics**: An innovative analytics platform that utilizes large language models (LLMs) for automated data analysis.\n- **Graffiti**: A minimalistic GraphQL framework, demonstrating his capability in modern web technologies.\n- **Particula**: A zero-config Express.js framework, along with its Fastify.js core and Next.js integration, highlighting his work in creating streamlined development tools.\n\nTim has also contributed to various utility projects, such as a music player for HumbleBundle soundtracks and a simple state management solution for React. His repositories indicate a strong focus on microservices architecture, with multiple projects utilizing Node.js and RabbitMQ.\n\nOverall, Tim's GitHub profile reflects a commitment to open-source development and a passion for creating tools that enhance web development and data management.",
116+
"usage": {
117+
"completionTokens": 249,
118+
"promptTokens": 607,
119+
"totalTokens": 856
120+
},
121+
"timingMs": 3012
122+
}
123+
]
124+
}
125+
]
126+
}

0 commit comments

Comments
 (0)