diff --git a/assets/connector/google_drive/document.png b/assets/connector/google_drive/document.png new file mode 100644 index 0000000..a9ad302 Binary files /dev/null and b/assets/connector/google_drive/document.png differ diff --git a/assets/connector/google_drive/form.png b/assets/connector/google_drive/form.png new file mode 100644 index 0000000..88746d3 Binary files /dev/null and b/assets/connector/google_drive/form.png differ diff --git a/assets/connector/google_drive/icon.png b/assets/connector/google_drive/icon.png new file mode 100644 index 0000000..4495af2 Binary files /dev/null and b/assets/connector/google_drive/icon.png differ diff --git a/assets/connector/google_drive/pdf.png b/assets/connector/google_drive/pdf.png new file mode 100644 index 0000000..ff0cc2f Binary files /dev/null and b/assets/connector/google_drive/pdf.png differ diff --git a/assets/connector/google_drive/presentation.png b/assets/connector/google_drive/presentation.png new file mode 100644 index 0000000..80799ec Binary files /dev/null and b/assets/connector/google_drive/presentation.png differ diff --git a/assets/connector/google_drive/spreadsheet.png b/assets/connector/google_drive/spreadsheet.png new file mode 100644 index 0000000..9735292 Binary files /dev/null and b/assets/connector/google_drive/spreadsheet.png differ diff --git a/assets/connector/yuque/doc/board.png b/assets/connector/yuque/board.png similarity index 100% rename from assets/connector/yuque/doc/board.png rename to assets/connector/yuque/board.png diff --git a/assets/connector/yuque/book/book.png b/assets/connector/yuque/book.png similarity index 100% rename from assets/connector/yuque/book/book.png rename to assets/connector/yuque/book.png diff --git a/assets/connector/yuque/doc/doc.png b/assets/connector/yuque/doc.png similarity index 100% rename from assets/connector/yuque/doc/doc.png rename to assets/connector/yuque/doc.png diff --git a/assets/connector/yuque/doc/sheet.png b/assets/connector/yuque/sheet.png similarity index 100% rename from assets/connector/yuque/doc/sheet.png rename to assets/connector/yuque/sheet.png diff --git a/assets/connector/yuque/doc/table.png b/assets/connector/yuque/table.png similarity index 100% rename from assets/connector/yuque/doc/table.png rename to assets/connector/yuque/table.png diff --git a/docs/content.en/docs/references/api/document.md b/docs/content.en/docs/references/api/document.md index 515e320..f1e6180 100644 --- a/docs/content.en/docs/references/api/document.md +++ b/docs/content.en/docs/references/api/document.md @@ -13,7 +13,7 @@ Below is the field description for the document. | **Field** | **Type** | **Description** | |-------------------------|--------------------|-----------------------------------------------------------------------------------------------------| -| `source` | `string` | The source of the document, e.g., `google_drive`. | +| `source` | `object` | The source of the document. | | `category` | `string` | Primary category of the document, e.g., `report`. | | `categories` | `array[string]` | List of categories the document belongs to, e.g., `["business", "quarterly_reports"]`. | | `cover` | `string` (URL) | URL to the cover image of the document. | @@ -31,7 +31,8 @@ Below is the field description for the document. | `owner.avatar` | `string` (URL) | URL to the owner's avatar image. | | `owner.username` | `string` | Username of the owner, e.g., `jdoe`. | | `owner.userid` | `string` | User ID of the owner, e.g., `user123`. | -| `metadata` | `object` | Additional metadata about the document. | +| `metadata` | `object` | Additional accessible metadata (e.g., file version, permissions). | +| `payload` | `object` | Additional store-only metadata (e.g., file binary data). | | `last_updated_by` | `object` | Information about the last user who updated the document. | | `last_updated_by.user` | `object` | Details about the user who last updated the document. | | `last_updated_by.user.avatar` | `string` (URL) | URL to the avatar of the last editor. | @@ -45,7 +46,11 @@ Below is the field description for the document. ```shell //request curl -H 'Content-Type: application/json' -XPOST http://localhost:2900/document/ -d '{ - "source": "google_drive", + "source": { + "type":"connector", + "name":"google_drive", + "id":"e806831dacc3", + }, "category": "report", "categories": ["business", "quarterly_reports"], "cover": "https://example.com/images/report_cover.jpg", @@ -111,7 +116,11 @@ curl -XGET http://localhost:2900/document/cso9vr3q50k38nobvmcg "id": "cso9vr3q50k38nobvmcg", "created": "2024-11-10T19:58:36.009086+08:00", "updated": "2024-11-10T19:58:36.009092+08:00", - "source": "google_drive", + "source": { + "type":"connector", + "name":"google_drive", + "id":"e806831dacc3", + } ...OMITTED... , "found": true diff --git a/docs/content.en/docs/references/api/search.md b/docs/content.en/docs/references/api/search.md index 462928b..2bf42fe 100644 --- a/docs/content.en/docs/references/api/search.md +++ b/docs/content.en/docs/references/api/search.md @@ -22,7 +22,11 @@ curl -XGET http://localhost:2900/query/_suggest\?query\=buss { "suggestion": "Q3 Business Report", "score": 0.99, - "source": "google_drive" + "source": { + "type":"connector", + "name":"google_drive", + "id":"e806831dacc3", + } } ] } diff --git a/modules/assistant/session.go b/modules/assistant/session.go index 16a0b6d..ceaefaa 100644 --- a/modules/assistant/session.go +++ b/modules/assistant/session.go @@ -161,7 +161,7 @@ func (h APIHandler) sendChatMessage(w http.ResponseWriter, req *http.Request, ps webSocketID:=req.Header.Get("WEBSOCKET-SESSION-ID") - log.Info(req.Header) + log.Trace(req.Header) sessionID := ps.MustGetParameter("session_id") var request MessageRequest diff --git a/modules/common/connector.go b/modules/common/connector.go new file mode 100644 index 0000000..55f4cc7 --- /dev/null +++ b/modules/common/connector.go @@ -0,0 +1,19 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package common + +type Connector struct { + CombinedFullText + Name string `json:"name,omitempty" elastic_mapping:"name:{type:keyword,copy_to:combined_fulltext}"` // Source of the document (e.g., "github", "google_drive", "dropbox") + Description string `json:"description,omitempty" elastic_mapping:"description:{type:keyword,copy_to:combined_fulltext}"` // Source of the document (e.g., "github", "google_drive", "dropbox") + Category string `json:"category,omitempty" elastic_mapping:"category:{type:keyword,copy_to:combined_fulltext}"` // Primary category of the document (e.g., "report", "article") + Icon string `json:"icon,omitempty" elastic_mapping:"icon:{enabled:false}"` // Thumbnail image URL, for preview purposes + Tags []string `json:"tags,omitempty" elastic_mapping:"tags:{type:keyword,copy_to:combined_fulltext}"` // Tags or keywords associated with the document, for easier retrieval + URL string `json:"url,omitempty" elastic_mapping:"url:{enabled:false}"` // Direct link to the document, if available + + Assets struct { + Icons map[string]string //icon_key -> URL + } +} diff --git a/modules/common/document.go b/modules/common/document.go index df2a483..e1aec7e 100644 --- a/modules/common/document.go +++ b/modules/common/document.go @@ -5,32 +5,54 @@ package common import ( - "infini.sh/framework/core/orm" "time" ) +type RichLabel struct { + Label string `json:"label,omitempty" elastic_mapping:"label:{type:keyword,copy_to:combined_fulltext}"` + Icon string `json:"icon,omitempty" elastic_mapping:"icon:{enabled:false}"` // Icon Key, need work with datasource's assets to get the icon url +} + +type DataSource struct { + Type string `json:"type,omitempty" elastic_mapping:"type:{type:keyword}"` // Type of the datasource, eg: connector + Name string `json:"name,omitempty" elastic_mapping:"name:{type:keyword}"` // Source of the document (e.g., "github", "google_drive", "dropbox") + ID string `json:"id,omitempty" elastic_mapping:"id:{type:keyword}"` // ID of this connector instance +} + type Document struct { - orm.ORMObjectBase // Embedding ORM base for persistence-related fields - - Source string `json:"source,omitempty" elastic_mapping:"source:{type:keyword,copy_to:combined_fulltext}"` // Source of the document (e.g., "github", "google_drive", "dropbox") - Category string `json:"category,omitempty" elastic_mapping:"category:{type:keyword,copy_to:combined_fulltext}"` // Primary category of the document (e.g., "report", "article") - Categories []string `json:"categories,omitempty" elastic_mapping:"categories:{type:keyword,copy_to:combined_fulltext}"` // Full hierarchy of categories, useful for detailed classification - Title string `json:"title,omitempty" elastic_mapping:"title:{type:text,copy_to:combined_fulltext,fields:{keyword: {type: keyword}, pinyin: {type: text, analyzer: pinyin_analyzer}}}"` // Document title - Summary string `json:"summary,omitempty" elastic_mapping:"summary:{type:text,copy_to:combined_fulltext}"` // Brief summary or description of the document - Lang string `json:"lang,omitempty" elastic_mapping:"lang:{type:keyword,copy_to:combined_fulltext}"` // Language code (e.g., "en", "fr") - Content string `json:"content,omitempty" elastic_mapping:"content:{type:text,copy_to:combined_fulltext}"` // Document content for full-text indexing - Icon string `json:"icon,omitempty" elastic_mapping:"icon:{enabled:false}"` // Thumbnail image URL, for preview purposes - Thumbnail string `json:"thumbnail,omitempty" elastic_mapping:"thumbnail:{enabled:false}"` // Thumbnail image URL, for preview purposes - Cover string `json:"cover,omitempty" elastic_mapping:"cover:{enabled:false}"` // Cover image URL, if applicable - Type string `json:"type,omitempty" elastic_mapping:"type:{type:keyword,copy_to:combined_fulltext}"` // Document type, such as PDF, Docx, etc. - Owner *UserInfo `json:"owner,omitempty" elastic_mapping:"owner:{type:object}"` // Document author or owner - Tags []string `json:"tags,omitempty" elastic_mapping:"tags:{type:keyword,copy_to:combined_fulltext}"` // Tags or keywords associated with the document, for easier retrieval - URL string `json:"url,omitempty" elastic_mapping:"url:{enabled:false}"` // Direct link to the document, if available - Size int `json:"size,omitempty" elastic_mapping:"size:{type:long}"` // File size in bytes, if applicable - Metadata map[string]interface{} `json:"metadata,omitempty" elastic_mapping:"metadata:{enabled:false}"` // Additional source-specific metadata (e.g., file version, permissions) - LastUpdatedBy *EditorInfo `json:"last_updated_by,omitempty" elastic_mapping:"last_updated_by:{type:object}"` // Struct containing last update information - - CombinedFullText string `json:"-" elastic_mapping:"combined_fulltext:{type:text,index_prefixes:{},index_phrases:true, analyzer:combined_text_analyzer }"` + CombinedFullText + + Source DataSource `json:"source,omitempty" elastic_mapping:"source:{type:object}"` // Source of the document + + Type string `json:"type,omitempty" elastic_mapping:"type:{type:keyword,copy_to:combined_fulltext}"` // Document type, such as PDF, Docx, etc. + + Category string `json:"category,omitempty" elastic_mapping:"category:{type:keyword,copy_to:combined_fulltext}"` // Primary category of the document (e.g., "report", "article") + Subcategory string `json:"subcategory,omitempty" elastic_mapping:"subcategory:{type:keyword,copy_to:combined_fulltext}"` // Secondary category of the document (e.g., "report", "article") + + //use categories for very complex hierarchy categories + Categories []string `json:"categories,omitempty" elastic_mapping:"categories:{type:keyword,copy_to:combined_fulltext}"` // Full hierarchy of categories, useful for detailed classification + + //use rich_categories for icon need to display for each category + RichCategories []RichLabel `json:"rich_categories,omitempty" elastic_mapping:"rich_categories:{type:object}"` // Full hierarchy of categories, useful for detailed classification, with icon decoration + + Title string `json:"title,omitempty" elastic_mapping:"title:{type:text,copy_to:combined_fulltext,fields:{keyword: {type: keyword}, pinyin: {type: text, analyzer: pinyin_analyzer}}}"` // Document title + Summary string `json:"summary,omitempty" elastic_mapping:"summary:{type:text,copy_to:combined_fulltext}"` // Brief summary or description of the document + + Lang string `json:"lang,omitempty" elastic_mapping:"lang:{type:keyword,copy_to:combined_fulltext}"` // Language code (e.g., "en", "fr") + Content string `json:"content,omitempty" elastic_mapping:"content:{type:text,copy_to:combined_fulltext}"` // Document content for full-text indexing + + Icon string `json:"icon,omitempty" elastic_mapping:"icon:{enabled:false}"` // Icon Key, need work with datasource's assets to get the icon url, if it is a full url, then use it directly + Thumbnail string `json:"thumbnail,omitempty" elastic_mapping:"thumbnail:{enabled:false}"` // Thumbnail image URL, for preview purposes + Cover string `json:"cover,omitempty" elastic_mapping:"cover:{enabled:false}"` // Cover image URL, if applicable + + Owner *UserInfo `json:"owner,omitempty" elastic_mapping:"owner:{type:object}"` // Document author or owner + + Tags []string `json:"tags,omitempty" elastic_mapping:"tags:{type:keyword,copy_to:combined_fulltext}"` // Tags or keywords associated with the document, for easier retrieval + URL string `json:"url,omitempty" elastic_mapping:"url:{enabled:false}"` // Direct link to the document, if available + Size int `json:"size,omitempty" elastic_mapping:"size:{type:long}"` // File size in bytes, if applicable + + LastUpdatedBy *EditorInfo `json:"last_updated_by,omitempty" elastic_mapping:"last_updated_by:{type:object}"` // Struct containing last update information + } type EditorInfo struct { diff --git a/modules/common/fulltext.go b/modules/common/fulltext.go new file mode 100644 index 0000000..e467632 --- /dev/null +++ b/modules/common/fulltext.go @@ -0,0 +1,16 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package common + +import "infini.sh/framework/core/orm" + + +type CombinedFullText struct { + orm.ORMObjectBase // Embedding ORM base for persistence-related fields + CombinedFullText string `json:"-" elastic_mapping:"combined_fulltext:{type:text,index_prefixes:{},index_phrases:true, analyzer:combined_text_analyzer }"` + + Metadata map[string]interface{} `json:"metadata,omitempty" elastic_mapping:"metadata:{type:object}"` // Additional accessible metadata (e.g., file version, permissions) + Payload map[string]interface{} `json:"payload,omitempty" elastic_mapping:"payload:{enabled:false}"` // Additional store-only metadata (e.g., file binary data) +} diff --git a/modules/search/search.go b/modules/search/search.go index e6018f0..731fbe3 100644 --- a/modules/search/search.go +++ b/modules/search/search.go @@ -83,11 +83,11 @@ func (h APIHandler) search(w http.ResponseWriter, req *http.Request, ps httprout templatedQuery := orm.TemplatedQuery{} templatedQuery.TemplateID = "coco-query-string" templatedQuery.Parameters = util.MapStr{ - "from": from, - "size": size, - "field": field, - "query": query, - "source": strings.Split(source, ","), + "from": from, + "size": size, + "field": field, + "query": query, + "source": strings.Split(source, ","), } q.TemplatedQuery = &templatedQuery } else { diff --git a/plugins/connectors/google_drive/files.go b/plugins/connectors/google_drive/files.go index a7b4749..786927e 100644 --- a/plugins/connectors/google_drive/files.go +++ b/plugins/connectors/google_drive/files.go @@ -18,6 +18,22 @@ import ( "time" ) +func getIcon(fileType string) string { + switch fileType { + case "application/vnd.google-apps.document": + return "document" + case "application/vnd.google-apps.form": + return "form" + case "application/pdf": + return "pdf" + case "application/vnd.google-apps.presentation": + return "presentation" + case "application/vnd.google-apps.spreadsheet": + return "spreadsheet" + default: + return "default" // default fallback + } +} func (this *Plugin) startIndexingFiles(tenantID,userID string,tok *oauth2.Token) { var filesProcessed =0 @@ -56,7 +72,7 @@ func (this *Plugin) startIndexingFiles(tenantID,userID string,tok *oauth2.Token) log.Tracef("get last modified time: %v",lastModifiedTimeStr) - if lastModifiedTimeStr !=""{ + if lastModifiedTimeStr !=""{ //TODO, if the files are newly shared and with old timestamp and we may missed // Parse last indexed time parsedTime, err := time.Parse(time.RFC3339Nano, lastModifiedTimeStr) if err != nil { @@ -113,7 +129,11 @@ func (this *Plugin) startIndexingFiles(tenantID,userID string,tok *oauth2.Token) // Map Google Drive file to Document struct document := common.Document{ - Source: "google_drive", + Source: common.DataSource{ + //ID: "",//TODO + Name: "google_drive", + Type: "connector", + }, Title: i.Name, Summary: i.Description, Type: i.MimeType, @@ -124,30 +144,32 @@ func (this *Plugin) startIndexingFiles(tenantID,userID string,tok *oauth2.Token) UserName: i.Owners[0].DisplayName, UserID: i.Owners[0].EmailAddress, }, - Icon: i.IconLink, + Icon: getIcon(i.MimeType), Thumbnail: i.ThumbnailLink, - Metadata: util.MapStr{ - "drive_id": i.DriveId, - "file_id": i.Id, - "email": i.Owners[0].EmailAddress, - "file_extension": i.FileExtension, - "kind": i.Kind, - "shared": i.Shared, - "spaces": i.Spaces, - "starred": i.Starred, - "web_view_link": i.WebViewLink, - "labels": i.LabelInfo, - "parents": i.Parents, - "permissions": i.Permissions, - "permission_ids": i.PermissionIds, - "properties": i.Properties, - }, + } - document.ID = i.Id + document.ID = i.Id //add tenant namespace and then hash document.Created = createdAt document.Updated = updatedAt + document.Metadata= util.MapStr{ + "drive_id": i.DriveId, + "file_id": i.Id, + "email": i.Owners[0].EmailAddress, + "file_extension": i.FileExtension, + "kind": i.Kind, + "shared": i.Shared, + "spaces": i.Spaces, + "starred": i.Starred, + "web_view_link": i.WebViewLink, + "labels": i.LabelInfo, + "parents": i.Parents, + "permissions": i.Permissions, + "permission_ids": i.PermissionIds, + "properties": i.Properties, + } + if i.LastModifyingUser != nil { document.LastUpdatedBy = &common.EditorInfo{ UserInfo: common.UserInfo{ @@ -159,9 +181,11 @@ func (this *Plugin) startIndexingFiles(tenantID,userID string,tok *oauth2.Token) } } + document.Payload= util.MapStr{} + // Handle optional fields if i.SharingUser != nil { - document.Metadata["sharingUser"] = common.UserInfo{ + document.Payload["sharingUser"] = common.UserInfo{ UserAvatar: i.SharingUser.PhotoLink, UserName: i.SharingUser.DisplayName, UserID: i.SharingUser.EmailAddress, @@ -169,11 +193,11 @@ func (this *Plugin) startIndexingFiles(tenantID,userID string,tok *oauth2.Token) } if i.VideoMediaMetadata != nil { - document.Metadata["video_metadata"] = i.VideoMediaMetadata + document.Payload["video_metadata"] = i.VideoMediaMetadata } if i.ImageMediaMetadata != nil { - document.Metadata["image_metadata"] = i.ImageMediaMetadata + document.Payload["image_metadata"] = i.ImageMediaMetadata } // Convert to JSON and push to queue diff --git a/plugins/connectors/yuque/collect.go b/plugins/connectors/yuque/collect.go index 79a63b2..4eef829 100644 --- a/plugins/connectors/yuque/collect.go +++ b/plugins/connectors/yuque/collect.go @@ -9,7 +9,6 @@ import ( log "github.com/cihub/seelog" "infini.sh/coco/modules/common" "infini.sh/framework/core/global" - "infini.sh/framework/core/orm" "infini.sh/framework/core/queue" "infini.sh/framework/core/util" "strings" @@ -32,30 +31,8 @@ func get(path, token string) *util.Result { return res } -func (this *Plugin) getIconLink(connector, category, iconType string) string { - // Retrieve the environment's configuration - env := global.Env() - tlsEnabled := env.SystemConfig.WebAppConfig.TLSConfig.TLSEnabled - domain := env.SystemConfig.WebAppConfig.Domain - - // Fallback to publish address if domain is empty - if domain == "" { - domain = env.SystemConfig.WebAppConfig.NetworkConfig.GetPublishAddr() - } - - // Construct the base URL - protocol := "http" - if tlsEnabled { - protocol = "https" - } - baseURL := protocol + "://" + domain - - icon := fmt.Sprintf("%s/assets/connector/%s/%s/%s.png", - baseURL, this.cleanupIconName(connector), this.cleanupIconName(category), this.cleanupIconName(iconType)) - return icon - - ////TODO cache and checking vfs and if not exists and then return the default icon - //return baseURL + "/assets/connector/default.png" +func (this *Plugin) getIconKey(category, iconType string) string { + return fmt.Sprintf("%v_%v",strings.TrimSpace(strings.ToLower(category)),strings.TrimSpace(strings.ToLower(iconType))) } func (this *Plugin) cleanupIconName(name string) string { @@ -82,6 +59,11 @@ func (this *Plugin) collect() { token := this.cfg.Token + if token==""{ + panic("invalid yuque token") + } + + //for groups only res := get("/api/v2/user", token) currentUser := struct { @@ -148,7 +130,11 @@ func (this *Plugin) collectBooks(login, token string) { //index books document := common.Document{ - Source: YuqueKey, + Source: common.DataSource{ + //ID: "",//TODO + Name: YuqueKey, + Type: "connector", + }, Title: bookDetail.Book.Name, Summary: bookDetail.Book.Description, Type: book.Type, @@ -159,20 +145,21 @@ func (this *Plugin) collectBooks(login, token string) { UserName: bookDetail.Book.User.Name, UserID: bookDetail.Book.User.Login, }, - Icon: this.getIconLink(YuqueKey, "book", bookDetail.Book.Type), + Icon: this.getIconKey("book", bookDetail.Book.Type), //Thumbnail: bookDetail.Book., - Metadata: util.MapStr{ - "public": bookDetail.Book.Public, - "slug": bookDetail.Book.Slug, - "creator_id": bookDetail.Book.CreatorID, - "user_id": bookDetail.Book.UserID, - "toc_yml": bookDetail.Book.TocYML, - "items_count": bookDetail.Book.ItemsCount, - "likes_count": bookDetail.Book.LikesCount, - "watches_count": bookDetail.Book.WatchesCount, - "namespace": bookDetail.Book.Namespace, - "user": bookDetail.Book.User, - }, + } + + document.Metadata= util.MapStr{ + "public": bookDetail.Book.Public, + "slug": bookDetail.Book.Slug, + "creator_id": bookDetail.Book.CreatorID, + "user_id": bookDetail.Book.UserID, + "toc_yml": bookDetail.Book.TocYML, + "items_count": bookDetail.Book.ItemsCount, + "likes_count": bookDetail.Book.LikesCount, + "watches_count": bookDetail.Book.WatchesCount, + "namespace": bookDetail.Book.Namespace, + "user": bookDetail.Book.User, } document.ID = util.MD5digest(fmt.Sprintf("%v-%v-%v", "test", "yuque-book", bookID)) @@ -257,7 +244,11 @@ func (this *Plugin) collectDocDetails(bookID int64, docID int64, token string) { if this.cfg.IndexingDocs && (doc.Doc.Public > 0 || (this.cfg.IncludePrivateDoc)) { //index doc document := common.Document{ - Source: YuqueKey, + Source: common.DataSource{ + //ID: "",//TODO + Name: YuqueKey, + Type: "connector", + }, Title: doc.Doc.Title, Summary: doc.Doc.Description, Cover: doc.Doc.Cover, @@ -270,30 +261,35 @@ func (this *Plugin) collectDocDetails(bookID int64, docID int64, token string) { UserName: doc.Doc.User.Name, UserID: doc.Doc.User.Login, }, - Icon: this.getIconLink(YuqueKey, "doc", doc.Doc.Type), + Icon: this.getIconKey("doc", doc.Doc.Type), Thumbnail: doc.Doc.Cover, - Metadata: util.MapStr{ - "public": doc.Doc.Public, - "slug": doc.Doc.Slug, - "user_id": doc.Doc.UserID, - "book_id": doc.Doc.BookID, - "last_editor_id": doc.Doc.LastEditorID, - "format": doc.Doc.Format, - "body_draft": doc.Doc.BodyDraft, - "body_html": doc.Doc.BodyHTML, - "body_sheet": doc.Doc.BodySheet, - "body_lake": doc.Doc.BodyLake, - "body_table": doc.Doc.BodyTable, - "status": doc.Doc.Status, - "likes_count": doc.Doc.LikesCount, - "read_count": doc.Doc.ReadCount, - "comments_count": doc.Doc.CommentsCount, - "word_count": doc.Doc.WordCount, - "user": doc.Doc.User, - "creator": doc.Doc.Creator, - "book": doc.Doc.Book, - "tags": doc.Doc.Tags, - }, + + } + + document.Metadata=util.MapStr{ + "public": doc.Doc.Public, + "slug": doc.Doc.Slug, + "user_id": doc.Doc.UserID, + "book_id": doc.Doc.BookID, + "last_editor_id": doc.Doc.LastEditorID, + "format": doc.Doc.Format, + "status": doc.Doc.Status, + "likes_count": doc.Doc.LikesCount, + "read_count": doc.Doc.ReadCount, + "comments_count": doc.Doc.CommentsCount, + "word_count": doc.Doc.WordCount, + "user": doc.Doc.User, + "creator": doc.Doc.Creator, + "book": doc.Doc.Book, + "tags": doc.Doc.Tags, + } + + document.Payload=util.MapStr{ + "body_draft": doc.Doc.BodyDraft, + "body_html": doc.Doc.BodyHTML, + "body_sheet": doc.Doc.BodySheet, + "body_lake": doc.Doc.BodyLake, + "body_table": doc.Doc.BodyTable, } document.ID = util.MD5digest(fmt.Sprintf("%v-%v-%v", "test", "yuque-doc", doc.Doc.ID)) @@ -343,19 +339,21 @@ func (this *Plugin) collectUsers(login, token string) { } document = common.Document{ - Source: YuqueKey, + Source: common.DataSource{ + //ID: "",//TODO + Name: YuqueKey, + Type: "connector", + }, Title: groupUser.User.Name, Summary: groupUser.User.Description, Type: docType, URL: fmt.Sprintf("https://infini.yuque.com/%v", groupUser.User.Login), - Icon: groupUser.User.AvatarURL, + Icon: groupUser.User.AvatarURL, //TODO save to local store Thumbnail: groupUser.User.AvatarURL, - Metadata: metadata, - ORMObjectBase: orm.ORMObjectBase{ - Created: &groupUser.User.CreatedAt, - Updated: &groupUser.User.UpdatedAt, - }, } + document.Created=&groupUser.User.CreatedAt + document.Updated=&groupUser.User.UpdatedAt + document.Metadata=metadata } else if groupUser.Group != nil && this.cfg.IndexingGroups { idPrefix, docType = "yuque-group", groupUser.Group.Type @@ -369,19 +367,21 @@ func (this *Plugin) collectUsers(login, token string) { } document = common.Document{ - Source: YuqueKey, + Source: common.DataSource{ + //ID: "",//TODO + Name: YuqueKey, + Type: "connector", + }, Title: groupUser.Group.Name, Summary: groupUser.Group.Description, Type: docType, URL: fmt.Sprintf("https://infini.yuque.com/%v", groupUser.Group.ID), - Icon: groupUser.Group.AvatarURL, + Icon: groupUser.Group.AvatarURL, //TODO save to local store Thumbnail: groupUser.Group.AvatarURL, - Metadata: metadata, - ORMObjectBase: orm.ORMObjectBase{ - Created: &groupUser.Group.CreatedAt, - Updated: &groupUser.Group.UpdatedAt, - }, } + document.Created=&groupUser.Group.CreatedAt + document.Updated=&groupUser.Group.UpdatedAt + document.Metadata=metadata } // Generate document ID and save