Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(document-reader): add Notion document reader support #274 #345

Merged
merged 4 commits into from
Jan 7, 2025
+767 −0
Merged

feat(document-reader): add Notion document reader support #274 #345

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ce14e3e
feat(document-reader): add Notion document reader support
brianxiadong Jan 3, 2025
5b05546
feat(document-reader):
brianxiadong Jan 6, 2025
431a122
feat(document-reader):
brianxiadong Jan 6, 2025
2ccf249
Merge branch 'main' into main
chickenlj Jan 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 94 additions & 0 deletions community/document-readers/notion-document-reader/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.cloud.ai</groupId>
<artifactId>spring-ai-alibaba</artifactId>
<version>${revision}</version>
<relativePath>../../../pom.xml</relativePath>
</parent>

<artifactId>notion-document-reader</artifactId>
<name>notion-document-reader</name>
<description>notion reader for Spring AI Alibaba</description>
<packaging>jar</packaging>
<url>https://github.com/alibaba/spring-ai-alibaba</url>
<scm>
<url>https://github.com/alibaba/spring-ai-alibaba</url>
<connection>git://github.com/alibaba/spring-ai-alibaba.git</connection>
<developerConnection>[email protected]:alibaba/spring-ai-alibaba.git</developerConnection>
</scm>

<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven-deploy-plugin.version>3.1.1</maven-deploy-plugin.version>
</properties>

<dependencies>

<dependency>
<groupId>com.alibaba.cloud.ai</groupId>
<artifactId>spring-ai-alibaba-core</artifactId>
<version>${project.parent.version}</version>
</dependency>

<!-- test dependencies -->
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-test</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-test</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-observation-test</artifactId>
<scope>test</scope>
</dependency>

</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>${maven-deploy-plugin.version}</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>

<repositories>
<repository>
<id>spring-milestones</id>
<name>Spring Milestones</name>
<url>https://repo.spring.io/milestone</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>

</project>
182 changes: 182 additions & 0 deletions ...ocument-reader/src/main/java/com/alibaba/cloud/ai/reader/notion/NotionDocumentReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
/*
* Copyright 2024-2025 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.cloud.ai.reader.notion;

import com.alibaba.fastjson.JSONObject;
import org.springframework.ai.document.Document;
import org.springframework.ai.document.DocumentReader;
import org.springframework.util.StringUtils;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.time.Instant;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* Notion Document Reader Implements DocumentReader interface to read content from Notion
*
* @author xiadong
* @since 2024-01-06
*/
public class NotionDocumentReader implements DocumentReader {

private final NotionResource notionResource;

private final JSONObject pageMetadata;

/**
* Constructor
* @param notionResource Notion resource
*/
public NotionDocumentReader(NotionResource notionResource) {
this.notionResource = notionResource;
this.pageMetadata = notionResource.getMetadata();
}

@Override
public List<Document> get() {
try {
// Read content from input stream
String content = readContent();

// Create metadata map
Map<String, Object> metadata = buildMetadata();

// Create and return document
return Collections.singletonList(new Document(content, metadata));
}
catch (IOException e) {
throw new RuntimeException("Failed to load document from Notion: " + e.getMessage(), e);
}
}

/**
* Build metadata map from Notion API response
*/
private Map<String, Object> buildMetadata() {
Map<String, Object> metadata = new HashMap<>();

// Add basic metadata
metadata.put(NotionResource.SOURCE, notionResource.getSource());
metadata.put("resourceType", notionResource.getResourceType().name());
metadata.put("resourceId", notionResource.getResourceId());

// Add metadata from Notion API
if (pageMetadata != null) {
// Creation and update times
String createdTime = pageMetadata.getString("created_time");
if (StringUtils.hasText(createdTime)) {
metadata.put("createdTime", Instant.parse(createdTime).toEpochMilli());
}

String lastEditedTime = pageMetadata.getString("last_edited_time");
if (StringUtils.hasText(lastEditedTime)) {
metadata.put("lastEditedTime", Instant.parse(lastEditedTime).toEpochMilli());
}

// Creator and last editor
JSONObject createdBy = pageMetadata.getJSONObject("created_by");
if (createdBy != null) {
String creatorName = createdBy.getString("name");
String creatorId = createdBy.getString("id");
if (StringUtils.hasText(creatorName)) {
metadata.put("createdBy", creatorName);
}
if (StringUtils.hasText(creatorId)) {
metadata.put("createdById", creatorId);
}
}

JSONObject lastEditedBy = pageMetadata.getJSONObject("last_edited_by");
if (lastEditedBy != null) {
String editorName = lastEditedBy.getString("name");
String editorId = lastEditedBy.getString("id");
if (StringUtils.hasText(editorName)) {
metadata.put("lastEditedBy", editorName);
}
if (StringUtils.hasText(editorId)) {
metadata.put("lastEditedById", editorId);
}
}

// URL
String url = pageMetadata.getString("url");
if (StringUtils.hasText(url)) {
metadata.put("url", url);
}

// Parent information
JSONObject parent = pageMetadata.getJSONObject("parent");
if (parent != null) {
String parentType = parent.getString("type");
if (StringUtils.hasText(parentType)) {
metadata.put("parentType", parentType);
String parentId = parent.getString(parentType + "_id");
if (StringUtils.hasText(parentId)) {
metadata.put("parentId", parentId);
}
}
}

// Icon
JSONObject icon = pageMetadata.getJSONObject("icon");
if (icon != null) {
String iconType = icon.getString("type");
String iconUrl = icon.getString("url");
if (StringUtils.hasText(iconType)) {
metadata.put("iconType", iconType);
}
if (StringUtils.hasText(iconUrl)) {
metadata.put("iconUrl", iconUrl);
}
}

// Cover
JSONObject cover = pageMetadata.getJSONObject("cover");
if (cover != null) {
String coverType = cover.getString("type");
String coverUrl = cover.getString("url");
if (StringUtils.hasText(coverType)) {
metadata.put("coverType", coverType);
}
if (StringUtils.hasText(coverUrl)) {
metadata.put("coverUrl", coverUrl);
}
}
}

return metadata;
}

/**
* Read content from input stream
*/
private String readContent() throws IOException {
StringBuilder content = new StringBuilder();
try (BufferedReader reader = new BufferedReader(new InputStreamReader(notionResource.getInputStream()))) {
String line;
while ((line = reader.readLine()) != null) {
content.append(line).append("\n");
}
}
return content.toString();
}

}
Loading
Loading