feat: propose grpc for queries in mvp (#11)

nebula_cli/src/main.rs

@@ -2,7 +2,7 @@
 
 use clap::{Parser, Subcommand};
 use nebula_common::{
-    client::{init_client, list_packages},
+    client::{init_client, list_packages, search_packages},
     configuration::cli::get_configuration,
 };
 
@@ -48,7 +48,7 @@ enum Command {
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let config = get_configuration()?;
     let reg_conf = config.remote_registry;
-    let mut client = init_client(reg_conf.port).await?;
+    let mut client = init_client(reg_conf.host, reg_conf.port).await?;
 
     let cli = CmdArgs::parse();
     match cli.cmd {
@@ -57,7 +57,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         //Command::Update => todo!(),
         //Command::Uninstall { all } => todo!(),
         Command::List { cached: _ } => println!("List: {:?}", list_packages(&mut client).await?),
-        //Command::Search { cached } => todo!(),
+        Command::Search { cached: _ } => {
+            println!("Search: {:?}", search_packages(&mut client, "cifar".into()).await?)
+        }
         //Command::Explore {} => todo!(),
         //Command::Sync => todo!(),
         //Command::Registry {} => todo!(),

nebula_common/proto/nebula.proto

@@ -1,24 +1,124 @@
 syntax = "proto3";
 
-package nebula;
+/**
+ *  The Nebula Registry interface in version v1 - this is not stable yet.
+ *
+ *  Focused on querying meta informaton
+ *  Open:
+ *  - Download needed files for installation
+ *  - Authentication
+ *  - Publishing models and datasets
+ */
+package nebula.v1;
 
-service NebulaRegistry {
+/**
+ *  A service responsible to provide package information to a client.
+ *  
+ *  By using this service a client should be able to receive all meta-data about a dataset or model
+ */
+service NebulaPackageQuery {
+    // Gets detailed information for one specific package
     rpc GetPackageInfo (PackageRequest) returns (PackageInfo);
-    rpc ListPackages (Empty) returns (PackageList);
+
+    // List all packages with very simple search criteria (substring search)
+    rpc ListPackages (ListPackagesRequest) returns (PackageList);
+
+    // Search packages applying several filters
+    rpc SearchPackages (SearchPackagesRequest) returns (PackageList);
+}
+
+// used for extended error reporting
+message ErrorDetail {
+    string reason = 1;
+    string hint = 2;
+}
+
+message DateRange {
+    optional string start = 1;  // YYYYMMDD - ISO 8601 format
+    optional string end = 2;    // YYYYMMDD - ISO 8601 format
+}
+
+enum PackageType {
+    BOTH    = 0;            // Retrive both datasets and models
+    DATASET = 1;            // Retrive only datasets
+    MODEL   = 2;            // Retrive only models
+}
+
+enum SearchKind {
+    RELAXED = 0;                // Searches for word-wise substrings in package name, authors and description in the same preference 
+    SUBSTR_PACKAGE_NAME = 1;    // Searches for substring in package name
+    SUBSTR_AUTHOR = 2;          // Searches for substring in author
+}
+
+enum SortOption {
+    CREATION_DATE   = 0;    // Sort by creation date
+    DOWNLOADS       = 1;    // Sort by download count descending
+    NAME            = 2;    // Sort Alphabetical by name
 }
 
+message SortParameter {
+    SortOption sort_by = 1;         // Sort option
+    optional bool descending = 2;   // Sort direction, default: ascending
+}
+
+message FieldOptions {
+    bool include_datapackage_json = 1;
+    bool include_preview_images = 2;
+}
+
+
+// message returns complete package info
 message PackageRequest {
-    string name = 1;
+    string search_query = 1;                // searches for EXACT package-name
+    PackageType package_type = 2;           // filters by dataset, model or both
+}
+
+message ListPackagesRequest {
+    FieldOptions field_options = 1;         // additional fields, like datapackage json or preview image
+    PackageType package_type = 2;           // filters by dataset, model or both
+    optional SortOption sort = 3;           // Sorting options
+    optional int32 limit = 4;               // Limit the number of results
+    optional int32 offset = 5;              // For pagination
+}
+
+message SearchPackagesRequest {
+    FieldOptions field_options = 1;         // additional fields, like datapackage json or preview image
+    string search_query = 2;                // General text search across multiple fields
+    PackageType package_type = 3;           // filters by dataset, model or both
+    repeated SortOption sort = 4;           // Sorting options, with support for different levels
+    optional int32 limit = 5;               // Limit the number of results
+    optional int32 offset = 6;              // For pagination
+    optional DateRange created_date = 7;    // Filter by creation date
+    optional DateRange updated_date = 8;    // Filter by last update date
+    optional SearchKind kind = 9;           // Search method (e.g., RELAXED, SUBSTR_PACKAGE_NAME)
+    repeated string authors = 10;           // Search by author(s)
+    optional int32 min_downloads = 11;      // Minimum number of downloads 
+    optional int32 max_downloads = 12;      // Maximum number of downloads
 }
 
 message PackageInfo {
     string name = 1;
     string version = 2;
     string description = 3;
+    uint64 download_size = 4;
+    uint64 installed_size = 5;
+    string license = 6;
+    optional string datapackage_json = 7;   // json string of datapackage json with delta extension
+    repeated string preview_images = 8;     // url or base64 encoded inline image
 }
 
 message PackageList {
-    repeated PackageInfo packages = 1;
+    repeated PackageInfo packages = 1; // List of package information
+    int32 total_count = 2;             // only set if pagiation was used
+    optional int32 limit = 3;          // only set if pagiation was used
+    optional int32 offset = 4;         // only set if pagiation was used
 }
 
-message Empty {}
+message Empty {}
+
+/**
+ * Service to publish packages on a Nebula registry, we'll do that later
+ */
+service NebulaPublisher {
+    rpc PublishPackage(Empty) returns (Empty);
+}

nebula_common/src/client/mod.rs

@@ -3,30 +3,63 @@
 use tonic::Request;
 use tonic::transport::Channel;
 
-use super::nebula_proto::nebula_registry_client::NebulaRegistryClient;
-use super::nebula_proto::{Empty, PackageInfo, PackageList, PackageRequest};
+use crate::server::{ListPackagesRequest, PackageType};
+
+use super::nebula_proto::nebula_package_query_client::NebulaPackageQueryClient;
+use super::nebula_proto::{PackageInfo, PackageList, PackageRequest, SearchPackagesRequest};
 
 pub async fn init_client(
+    host: String,
     port: u16,
-) -> Result<NebulaRegistryClient<Channel>, Box<dyn std::error::Error>> {
-    let client = NebulaRegistryClient::connect(format!("http://[127.0.0.1]:{}", port)).await?;
+) -> Result<NebulaPackageQueryClient<Channel>, Box<dyn std::error::Error>> {
+    let client = NebulaPackageQueryClient::connect(format!("http://{}:{}", host, port)).await?;
     Ok(client)
 }
 
 pub async fn list_packages(
-    client: &mut NebulaRegistryClient<Channel>,
+    client: &mut NebulaPackageQueryClient<Channel>,
 ) -> Result<PackageList, Box<dyn std::error::Error>> {
-    let request = Request::new(Empty {});
+    let request = Request::new(ListPackagesRequest {
+        field_options: None,
+        package_type: PackageType::Both as i32,
+        sort: None,
+        limit: Some(30),
+        offset: None,
+    });
     let response = client.list_packages(request).await?;
     Ok(response.into_inner())
 }
 
 pub async fn get_package_info(
-    client: &mut NebulaRegistryClient<Channel>,
+    client: &mut NebulaPackageQueryClient<Channel>,
     name: String,
 ) -> Result<PackageInfo, Box<dyn std::error::Error>> {
-    let request = Request::new(PackageRequest { name });
+    let request =
+        Request::new(PackageRequest { search_query: name, package_type: PackageType::Both as i32 });
     let response = client.get_package_info(request).await?;
 
     Ok(response.into_inner())
 }
+
+pub async fn search_packages(
+    client: &mut NebulaPackageQueryClient<Channel>,
+    query: String,
+) -> Result<PackageList, Box<dyn std::error::Error>> {
+    let request = Request::new(SearchPackagesRequest {
+        field_options: None,
+        search_query: query,
+        package_type: PackageType::Both as i32,
+        sort: vec![],
+        limit: None,
+        offset: None,
+        created_date: None,
+        updated_date: None,
+        kind: None,
+        authors: vec![],
+        min_downloads: None,
+        max_downloads: None,
+    });
+    let response = client.search_packages(request).await?;
+
+    Ok(response.into_inner())
+}

nebula_common/src/lib.rs

@@ -4,5 +4,9 @@ pub mod datapackage;
 pub mod server;
 
 pub mod nebula_proto {
-    tonic::include_proto!("nebula");
+    tonic::include_proto!("nebula.v1");
+}
+
+pub mod nebula_proto_fallback {
+    // include older version here
 }

nebula_common/src/server/endpoints.rs

@@ -1,21 +1,26 @@
-use super::nebula_registry_server::NebulaRegistry;
-use super::{Empty, PackageInfo, PackageList, PackageRequest};
+use super::nebula_package_query_server::NebulaPackageQuery;
+use super::{ListPackagesRequest, PackageInfo, PackageList, PackageRequest, SearchPackagesRequest};
 
 use tonic::{Request, Response, Status};
 
 #[derive(Debug, Default)]
-pub struct NebulaRegistryImpl {}
+pub struct NebulaPackageQueryMockImpl {}
 
 fn dummy_package() -> PackageInfo {
     PackageInfo {
         description: "Dataset image classification".to_string(),
         name: "cifar10".to_string(),
         version: "2".to_string(),
+        download_size: 0,
+        installed_size: 0,
+        license: "todo".into(),
+        datapackage_json: None,
+        preview_images: vec![],
     }
 }
 
 #[tonic::async_trait]
-impl NebulaRegistry for NebulaRegistryImpl {
+impl NebulaPackageQuery for NebulaPackageQueryMockImpl {
     async fn get_package_info(
         &self,
         request: Request<PackageRequest>,
@@ -27,12 +32,24 @@ impl NebulaRegistry for NebulaRegistryImpl {
 
     async fn list_packages(
         &self,
-        request: Request<Empty>,
+        request: Request<ListPackagesRequest>,
     ) -> Result<Response<PackageList>, Status> {
         println!("Got a request: {:?}", request);
 
-        let response_body = PackageList { packages: vec![dummy_package()] };
+        let response_body = PackageList {
+            packages: vec![dummy_package()],
+            total_count: 1,
+            limit: None,
+            offset: None,
+        };
 
         Ok(Response::new(response_body))
     }
+
+    async fn search_packages(
+        &self,
+        _request: Request<SearchPackagesRequest>,
+    ) -> Result<Response<PackageList>, Status> {
+        Err(Status::internal("not implemented"))
+    }
 }

nebula_common/src/server/mod.rs

@@ -1,9 +1,9 @@
 //! The server-side implementation of the nebula-registry RPC protocol
 
-pub use super::nebula_proto::nebula_registry_server::NebulaRegistryServer;
+pub use super::nebula_proto::nebula_package_query_server::NebulaPackageQueryServer;
 
 #[allow(unused)]
 pub(crate) use super::nebula_proto::*;
 
 pub mod endpoints;
-pub use endpoints::NebulaRegistryImpl;
+pub use endpoints::NebulaPackageQueryMockImpl;

nebula_registry/src/main.rs

@@ -2,7 +2,7 @@ use tonic::transport::Server;
 
 use nebula_common::{
     configuration::registry::get_configuration,
-    server::{NebulaRegistryImpl, NebulaRegistryServer},
+    server::{NebulaPackageQueryMockImpl, NebulaPackageQueryServer},
 };
 
 #[tokio::main]
@@ -11,10 +11,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let app_conf = config.application;
 
     let addr = format!("{}:{}", app_conf.host, app_conf.port).parse()?;
-    let registry = NebulaRegistryImpl::default();
+    let registry = NebulaPackageQueryMockImpl::default();
 
     println!("Nebula Registry v0.1.0 - running on: '{}'", addr);
-    Server::builder().add_service(NebulaRegistryServer::new(registry)).serve(addr).await?;
+    Server::builder().add_service(NebulaPackageQueryServer::new(registry)).serve(addr).await?;
 
     Ok(())
 }