Merge pull request #49 from dcSpark/nico/add_json_jinja2_to_markdown

tool json jinja2 to md

apps/shinkai-tool-json-to-md/jest.config.ts

@@ -0,0 +1,6 @@
+/* eslint-disable */
+export default {
+  displayName: '@shinkai_protocol/shinkai-tool-json-to-md',
+  preset: '../../jest.preset.js',
+  coverageDirectory: '../../coverage/apps/shinkai-tool-json-to-md',
+};

apps/shinkai-tool-json-to-md/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "@shinkai_protocol/shinkai-tool-json-to-md",
+  "type": "commonjs"
+}

apps/shinkai-tool-json-to-md/project.json

@@ -0,0 +1,30 @@
+{
+  "name": "@shinkai_protocol/shinkai-tool-json-to-md",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "apps/shinkai-tool-json-to-md/src",
+  "projectType": "library",
+  "tags": ["tool"],
+  "targets": {
+    "build": {
+      "executor": "nx:run-commands",
+      "defaultConfiguration": "production",
+      "options": {
+        "command": "npx ts-node scripts/tool-bundler.ts --entry ./apps/shinkai-tool-json-to-md/src/index.ts --outputFolder ./dist/apps/shinkai-tool-json-to-md"
+      },
+      "configurations": {
+        "development": {},
+        "production": {}
+      }
+    },
+    "lint": {
+      "executor": "@nx/linter:eslint",
+      "outputs": ["{options.outputFile}"],
+      "options": {
+        "lintFilePatterns": [
+          "apps/shinkai-tool-json-to-md/**/*.ts",
+          "apps/shinkai-tool-json-to-md/package.json"
+        ]
+      }
+    }
+  }
+}

apps/shinkai-tool-json-to-md/src/index.test.ts

@@ -0,0 +1,202 @@
+import { Tool } from '../src/index';
+
+// Sample input JSON
+const inputJson = JSON.stringify({
+  answersInText: [
+    {
+      citation_id: 0,
+      relevantTextFromDocument:
+        'SpaceX is the first private company to develop a liquid-propellant rocket that has reached orbit; to launch, orbit, and recover a spacecraft; to send a spacecraft to the International Space Station; a...',
+    },
+    {
+      citation_id: 1,
+      relevantTextFromDocument:
+        "It is also the first organization of any type to achieve a vertical propulsive landing of an orbital rocket booster and the first to reuse such a booster. The company's Falcon 9 rockets have landed an...",
+    },
+    {
+      citation_id: 2,
+      relevantTextFromDocument:
+        'SpaceX developed its first orbital launch vehicle, the Falcon 1, with internal funding.',
+    },
+    {
+      citation_id: 3,
+      relevantTextFromDocument:
+        'The Falcon 1 was an expendable two-stage-to-orbit small-lift launch vehicle.',
+    },
+    {
+      citation_id: 4,
+      relevantTextFromDocument:
+        'The total development cost of Falcon 1 was approximately $90 million to $100 million.',
+    },
+    {
+      citation_id: 5,
+      relevantTextFromDocument:
+        'The financial situation started to turn around with the first successful launch achieved on the fourth attempt on 28 September 2008.',
+    },
+    {
+      citation_id: 6,
+      relevantTextFromDocument:
+        "The first operational Dragon spacecraft was launched in December 2010 aboard COTS Demo Flight 1, the Falcon 9's second flight, and safely returned to Earth after two orbits, completing all its mission...",
+    },
+    {
+      citation_id: 7,
+      relevantTextFromDocument:
+        'In May 2012, with the Dragon C2+ launch, Dragon became the first commercial spacecraft to deliver cargo to the International Space Station.',
+    },
+    {
+      citation_id: 8,
+      relevantTextFromDocument:
+        'SpaceX first achieved a successful landing and recovery of a first stage in December 2015 with Falcon 9 Flight 20.',
+    },
+    {
+      citation_id: 9,
+      relevantTextFromDocument:
+        'In April 2016, the company achieved the first successful landing on the autonomous spaceport drone ship (ASDS) Of Course I Still Love You in the Atlantic Ocean.',
+    },
+    {
+      citation_id: 10,
+      relevantTextFromDocument:
+        'In March 2017, SpaceX launched a returned Falcon 9 for the SES-10 satellite. This was the first time a re-launch of a payload-carrying orbital rocket went back to space.',
+    },
+    {
+      citation_id: 11,
+      relevantTextFromDocument:
+        'A significant milestone was achieved in May 2020, when SpaceX successfully launched two NASA astronauts (Doug Hurley and Bob Behnken) into orbit on a Crew Dragon spacecraft during Crew Dragon Demo-2, ...',
+    },
+  ],
+  answer: {
+    introduction: {
+      sentences: [
+        'SpaceX has achieved several notable milestones in space exploration and technology.',
+      ],
+    },
+    body: [
+      {
+        sentences: [
+          'SpaceX is the first private company to develop a liquid-propellant rocket that has reached orbit; to launch, orbit, and recover a spacecraft; to send a spacecraft to the International Space Station; and to send astronauts to the International Space Station.[0]',
+          'It is also the first organization of any type to achieve a vertical propulsive landing of an orbital rocket booster and the first to reuse such a booster.[1]',
+          'The company currently produces and operates the Falcon 9 and Falcon Heavy rockets along with the Dragon spacecraft.[0]',
+        ],
+      },
+      {
+        sentences: [
+          "The Falcon 1, developed with internal funding, was SpaceX's first orbital launch vehicle, and it was an expendable two-stage-to-orbit small-lift launch vehicle that achieved its first successful launch in 2008.[2]",
+          "In December 2010, the first operational Dragon spacecraft launched aboard Falcon 9's second flight and safely returned to Earth after two orbits, completing all its mission objectives.[6]",
+          'In May 2012, Dragon became the first commercial spacecraft to deliver cargo to the ISS.[7]',
+        ],
+      },
+      {
+        sentences: [
+          'In December 2015, SpaceX achieved a successful landing and recovery of a first stage with Falcon 9 Flight 20, marking the first such achievement for any organization.[8]',
+          'In April 2016, they achieved the first successful landing on the autonomous spaceport drone ship (ASDS) Of Course I Still Love You in the Atlantic Ocean.[9]',
+          'In March 2017, SpaceX successfully launched a returned Falcon 9 for the SES-10 satellite, the first re-launch of a payload-carrying orbital rocket.[10]',
+        ],
+      },
+      {
+        sentences: [
+          'A significant milestone was achieved in May 2020, when SpaceX successfully launched two NASA astronauts into orbit on a Crew Dragon spacecraft during Crew Dragon Demo-2, making them the first private company to send astronauts to the International Space Station and marking the first crewed orbital launch from American soil in 9 years.[11]',
+        ],
+      },
+    ],
+    conclusion: [
+      {
+        sentences: [
+          "Overall, SpaceX's advancements in rocket reusability, successful commercial missions, and crewed spaceflights have solidified their position as a leader in the aerospace industry.",
+        ],
+      },
+    ],
+  },
+});
+
+// Nunjucks template
+const template = `
+# Introduction
+
+{%- for sentence in answer.introduction.sentences %}
+{{ sentence }}
+{%- endfor %}
+
+# Body
+
+{%- for section in answer.body %}
+## Section {{ loop.index }}
+
+{%- for sentence in section.sentences %}
+{{ sentence }}
+{%- endfor %}
+{%- endfor %}
+
+# Conclusion
+
+{%- for section in answer.conclusion %}
+{{ section.sentences[0] }}
+{%- endfor %}
+
+# Citations
+
+{%- for citation in answersInText %}
+[{{ citation.citation_id }}]: {{ citation.relevantTextFromDocument }}
+{%- endfor %}`;
+
+// Expected Markdown output
+const expectedMarkdown = `# Introduction
+SpaceX has achieved several notable milestones in space exploration and technology.
+
+# Body
+## Section 1
+SpaceX is the first private company to develop a liquid-propellant rocket that has reached orbit; to launch, orbit, and recover a spacecraft; to send a spacecraft to the International Space Station; and to send astronauts to the International Space Station.[0]
+It is also the first organization of any type to achieve a vertical propulsive landing of an orbital rocket booster and the first to reuse such a booster.[1]
+The company currently produces and operates the Falcon 9 and Falcon Heavy rockets along with the Dragon spacecraft.[0]
+## Section 2
+The Falcon 1, developed with internal funding, was SpaceX's first orbital launch vehicle, and it was an expendable two-stage-to-orbit small-lift launch vehicle that achieved its first successful launch in 2008.[2]
+In December 2010, the first operational Dragon spacecraft launched aboard Falcon 9's second flight and safely returned to Earth after two orbits, completing all its mission objectives.[6]
+In May 2012, Dragon became the first commercial spacecraft to deliver cargo to the ISS.[7]
+## Section 3
+In December 2015, SpaceX achieved a successful landing and recovery of a first stage with Falcon 9 Flight 20, marking the first such achievement for any organization.[8]
+In April 2016, they achieved the first successful landing on the autonomous spaceport drone ship (ASDS) Of Course I Still Love You in the Atlantic Ocean.[9]
+In March 2017, SpaceX successfully launched a returned Falcon 9 for the SES-10 satellite, the first re-launch of a payload-carrying orbital rocket.[10]
+## Section 4
+A significant milestone was achieved in May 2020, when SpaceX successfully launched two NASA astronauts into orbit on a Crew Dragon spacecraft during Crew Dragon Demo-2, making them the first private company to send astronauts to the International Space Station and marking the first crewed orbital launch from American soil in 9 years.[11]
+
+# Conclusion
+Overall, SpaceX's advancements in rocket reusability, successful commercial missions, and crewed spaceflights have solidified their position as a leader in the aerospace industry.
+
+# Citations
+[0]: SpaceX is the first private company to develop a liquid-propellant rocket that has reached orbit; to launch, orbit, and recover a spacecraft; to send a spacecraft to the International Space Station; a...
+[1]: It is also the first organization of any type to achieve a vertical propulsive landing of an orbital rocket booster and the first to reuse such a booster. The company's Falcon 9 rockets have landed an...
+[2]: SpaceX developed its first orbital launch vehicle, the Falcon 1, with internal funding.
+[3]: The Falcon 1 was an expendable two-stage-to-orbit small-lift launch vehicle.
+[4]: The total development cost of Falcon 1 was approximately $90 million to $100 million.
+[5]: The financial situation started to turn around with the first successful launch achieved on the fourth attempt on 28 September 2008.
+[6]: The first operational Dragon spacecraft was launched in December 2010 aboard COTS Demo Flight 1, the Falcon 9's second flight, and safely returned to Earth after two orbits, completing all its mission...
+[7]: In May 2012, with the Dragon C2+ launch, Dragon became the first commercial spacecraft to deliver cargo to the International Space Station.
+[8]: SpaceX first achieved a successful landing and recovery of a first stage in December 2015 with Falcon 9 Flight 20.
+[9]: In April 2016, the company achieved the first successful landing on the autonomous spaceport drone ship (ASDS) Of Course I Still Love You in the Atlantic Ocean.
+[10]: In March 2017, SpaceX launched a returned Falcon 9 for the SES-10 satellite. This was the first time a re-launch of a payload-carrying orbital rocket went back to space.
+[11]: A significant milestone was achieved in May 2020, when SpaceX successfully launched two NASA astronauts (Doug Hurley and Bob Behnken) into orbit on a Crew Dragon spacecraft during Crew Dragon Demo-2, ...`;
+
+test('exists definition', async () => {
+  const tool = new Tool({
+    only_system: false,
+  });
+  const definition = tool.getDefinition();
+  expect(definition).toBeInstanceOf(Object);
+});
+
+test('Tool converts JSON to Markdown correctly', async () => {
+  const tool = new Tool({
+    only_system: false,
+  });
+
+  const params = {
+    message: inputJson,
+    template: template.trim(),
+  };
+
+  const result = await tool.run(params);
+  console.log('result: ');
+  console.log(result);
+  expect(result).toHaveProperty('data.message');
+  expect(result.data.message.trim()).toBe(expectedMarkdown.trim());
+});
+

apps/shinkai-tool-json-to-md/src/index.ts

@@ -0,0 +1,69 @@
+import { BaseTool, RunResult } from '@shinkai_protocol/shinkai-tools-builder';
+import { ToolDefinition } from 'libs/shinkai-tools-builder/src/tool-definition';
+import * as nunjucks from 'nunjucks';
+
+type Config = {
+  only_system: boolean;
+};
+type Params = {
+  message: string;
+  template: string;
+};
+type Result = { message: string };
+
+export class Tool extends BaseTool<Config, Params, Result> {
+  definition: ToolDefinition<Config, Params, Result> = {
+    id: 'shinkai-tool-json-to-md',
+    name: 'Shinkai: json-to-md',
+    description: 'Converts JSON to Markdown using a Nunjucks (Jinja2-like) template',
+    author: 'Shinkai',
+    keywords: ['json-to-md', 'shinkai', 'nunjucks', 'jinja2', 'templating'],
+    configurations: {
+      type: 'object',
+      properties: {
+        only_system: { type: 'boolean' },
+      },
+      required: ['only_system'],
+    },
+    parameters: {
+      type: 'object',
+      properties: {
+        message: { type: 'string' },
+        template: { type: 'string' },
+      },
+      required: ['message', 'template'],
+    },
+    result: {
+      type: 'object',
+      properties: {
+        message: { type: 'string' },
+      },
+      required: ['message'],
+    },
+  };
+
+  async run(params: Params): Promise<RunResult<Result>> {
+    try {
+      // Parse the input JSON string
+      const inputData = JSON.parse(params.message);
+
+      // Check if a template is provided
+      if (!params.template) {
+        throw new Error('A template parameter is required to map the JSON to Markdown.');
+      }
+
+      const template = params.template;
+
+      // Configure Nunjucks (you can set options as needed)
+      nunjucks.configure({ autoescape: false });
+
+      // Render the template with the input data
+      const markdown = nunjucks.renderString(template, inputData);
+
+      return Promise.resolve({ data: { message: markdown } });
+    } catch (error) {
+      console.error('Error processing input:', error);
+      return Promise.reject(new Error('Invalid input JSON or template'));
+    }
+  }
+}

apps/shinkai-tool-json-to-md/tsconfig.app.json

@@ -0,0 +1,4 @@
+{
+  "extends": "./tsconfig.json",
+  "include": ["./src/**/*.ts"]
+}

apps/shinkai-tool-json-to-md/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+
+  },
+  "include": [
+    "./src/**/*.ts",
+    "webpack.config.ts"
+  ],
+}

apps/shinkai-tool-json-to-md/tsconfig.spec.json

@@ -0,0 +1,14 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "module": "commonjs",
+    "types": ["jest", "node"]
+  },
+  "include": [
+    "jest.config.ts",
+    "src/**/*.test.ts",
+    "src/**/*.spec.ts",
+    "src/**/*.d.ts"
+  ]
+}

libs/shinkai-tools-runner/src/built_in_tools.rs

@@ -237,6 +237,16 @@ lazy_static! {
                 .unwrap(),
             )),
         );
+        m.insert(
+            "shinkai-tool-json-to-md",
+            &*Box::leak(Box::new(
+                serde_json::from_str::<ToolDefinition>(include_str!(concat!(
+                    env!("CARGO_MANIFEST_DIR"),
+                    "/tools/shinkai-tool-json-to-md/definition.json"
+                )))
+                .unwrap(),
+            )),
+        );
         // ntim: New tools will be inserted here, don't remove this comment
         m
     };