Fetching data with channels

koucha/Cargo.lock

@@ -8,6 +8,15 @@ version = "0.2.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
 
+[[package]]
+name = "android_system_properties"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "atoi"
 version = "2.0.0"
@@ -200,7 +209,11 @@ version = "0.4.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118"
 dependencies = [
+ "iana-time-zone",
+ "js-sys",
  "num-traits",
+ "wasm-bindgen",
+ "windows-link",
 ]
 
 [[package]]
@@ -836,6 +849,30 @@ dependencies = [
  "windows-registry",
 ]
 
+[[package]]
+name = "iana-time-zone"
+version = "0.1.64"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "iana-time-zone-haiku",
+ "js-sys",
+ "log",
+ "wasm-bindgen",
+ "windows-core",
+]
+
+[[package]]
+name = "iana-time-zone-haiku"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "icu_collections"
 version = "2.1.1"
@@ -1023,6 +1060,7 @@ name = "koucha"
 version = "0.1.0"
 dependencies = [
  "axum",
+ "chrono",
  "reqwest",
  "rss",
  "sqlx",
@@ -2547,6 +2585,41 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "windows-core"
+version = "0.62.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
+dependencies = [
+ "windows-implement",
+ "windows-interface",
+ "windows-link",
+ "windows-result",
+ "windows-strings",
+]
+
+[[package]]
+name = "windows-implement"
+version = "0.60.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "windows-interface"
+version = "0.59.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "windows-link"
 version = "0.2.1"

koucha/Cargo.toml

@@ -9,3 +9,4 @@ reqwest = "0.13.1"
 rss = "2.0.12"
 tokio = { version = "1.49.0", features = ["full"] }
 sqlx = { version = "0.8.6", features = [ "runtime-tokio", "sqlite" ] }
+chrono = "0.4.43"

koucha/src/lib.rs

@@ -1,9 +1,12 @@
-use std::error::Error;
+use std::{
-use reqwest::{
+  error::Error,
-  IntoUrl,
+  hash::{Hash, Hasher},
-  Client,
+};
+use reqwest::Url;
+use chrono::{
+  Utc,
+  DateTime,
 };
-use rss::Channel as RawChannel;
 
 type Result<T> = std::result::Result<T, Box<dyn Error>>;
 
@@ -26,33 +29,42 @@ impl AdapterOptions {
   pub async fn create(self) -> Result<Adapter> {
     let db = sqlx::sqlite::SqlitePoolOptions::new()
       .connect(&self.database_url).await?;
+    let client = reqwest::Client::new();
 
-    Ok(Adapter { db })
+    Ok(Adapter { db, client })
   }
 }
 
 pub struct Adapter {
   db: sqlx::SqlitePool,
+  client: reqwest::Client,
 }
 
 impl Adapter {
   pub async fn get_all_users(&self) -> Result<Vec<User>> {
-      let users_query = sqlx::query!("SELECT id, name FROM users")
+    let users = sqlx::query_as!(
-        .fetch_all(&self.db).await?;
+      User,
+      "SELECT id, name FROM users"
+    ).fetch_all(&self.db).await?;
 
-      let mut all_users: Vec<User> = Vec::with_capacity(users_query.len());
+    Ok(users)
-
-      for user in users_query {
-        all_users.push(User {
-          id: user.id,
-          name: user.name,
-        })
-      }
-
-      Ok(all_users)
   }
 
+  // pub async fn update_channels(&self) -> Result<()> {
+  //
+  // }
+  // 
+  // async fn get_all_channels(&self) -> Result<Vec<impl Channel>> {
+  //   let users = sqlx::query_as!(
+  //     Channel,
+  //     "SELECT id FROM channels"
+  //   ).fetch_all(&self.db).await?;
+  //
+  //   Ok(users)
+  // }
+
   fn get_pool(&self) -> &sqlx::SqlitePool { &self.db }
+  fn get_client(&self) -> &reqwest::client { &self.client }
 }
 
 pub struct User {
@@ -61,15 +73,15 @@ pub struct User {
 }
 
 impl User {
-  pub async fn get_by_id(adapter: &Adapter, id: i64) -> Result<Self> {
+  // async fn get_by_id(adapter: &Adapter, id: i64) -> Result<Self> {
-    let user = sqlx::query!("SELECT name FROM users WHERE id = ?", id)
+  //   let user = sqlx::query!("SELECT name FROM users WHERE id = ?", id)
-      .fetch_one(adapter.get_pool()).await?;
+  //     .fetch_one(adapter.get_pool()).await?;
-
+  //
-    Ok(Self {
+  //   Ok(Self {
-      id: id,
+  //     id: id,
-      name: user.name,
+  //     name: user.name,
-    })
+  //   })
-  }
+  // }
 
   pub async fn create(adapter: &Adapter, name: &str) -> Result<Self> {
     let result = sqlx::query!("INSERT INTO users (name) VALUES (?)", name)
@@ -93,21 +105,123 @@ impl User {
     Ok(())
   }
 
+  pub async fn get_feeds(&self, adapter: &Adapter) -> Result<Vec<Feed>> {
+    let feeds = sqlx::query_as!(
+        Feed,
+        "SELECT id FROM feeds WHERE user_id = ?",
+        self.id
+      ).fetch_all(adapter.get_pool()).await?;
+
+    Ok(feeds)
+  }
+
   pub fn name(&self) -> &str { &self.name }
-  pub fn id(&self) -> i64 { self.id }
+}
+
+pub struct Feed {
+  id: i64,
+}
+
+impl Feed {
+  pub async fn get_items(
+    &self, adapter: &Adapter, limit: u8, offset: u32) -> Result<Vec<Item>> {
+    let items = sqlx::query_as!(
+      Item,
+      "SELECT item_id as id FROM feed_items 
+       WHERE feed_id = ? AND archived = FALSE
+       ORDER BY score DESC
+       LIMIT ? OFFSET ?",
+      self.id, limit, offset
+    ).fetch_all(adapter.get_pool()).await?;
+
+    Ok(items)
+  }
+
+  pub async fn get_channels(&self, adapter: &Adapter) -> Result<Vec<Channel>> {
+    let db_channels = sqlx::query!(
+      "SELECT c.id as `id!`, c.title, c.link, c.description, c.last_fetched
+       FROM channels c
+       JOIN feed_channels fc on c.id = fc.channel_id
+       WHERE fc.feed_id = ?",
+      self.id
+    ).fetch_all(adapter.get_pool()).await?;
+    let mut channels = Vec::with_capacity(db_channels.len());
+    for db_channel in db_channels {
+      channels.push(Channel {
+        id: db_channel.id,
+        title: db_channel.title,
+        link: Url::parse(&db_channel.link)?,
+        description: db_channel.description,
+        last_fetched: db_channel.last_fetched.as_deref()
+          .map(DateTime::parse_from_rfc2822)
+          .transpose()?
+          .map(|dt| dt.with_timezone(&Utc)),
+      })
+    }
+    Ok(channels)
+  }
 }
 
 pub struct Channel {
-  pub channel: rss::Channel,
+  id: i64,
+  title: String,
+  link: Url,
+  description: Option<String>,
+  last_fetched: Option<DateTime<Utc>>,
 }
 
-pub async fn fetch_channel<T: IntoUrl>(
+impl Channel {
-  client: &Client, url: T) -> Result<Channel> {
+  pub async fn fetch(mut self, adapter: &Adapter) -> Result<Self> {
-  let content = client.get(url)
+    let bytestream = adapter.get_client().get(self.link.clone())
-    .send().await?
+      .send().await?
-    .bytes().await?;
+      .bytes().await?;
 
-  let raw_channel = RawChannel::read_from(&content[..])?;
+    let rss_channel = rss::Channel::read_from(&bytestream[..])?;
-  println!("{}", raw_channel.title);
+    self.title = rss_channel.title;
-  Ok(Channel { channel: raw_channel })
+    self.link = Url::parse(&rss_channel.link)?;
+    self.description = Some(rss_channel.description);
+    let now = Utc::now();
+    self.last_fetched = Some(now);
+
+    sqlx::query!(
+      "UPDATE channels
+       SET title = ?, link = ?, description = ?, 
+        last_fetched = ?
+       WHERE id = ?",
+      self.title, self.link.as_str(), self.description, now.to_rfc2822(),
+      self.id
+    ).execute(adapter.get_pool()).await?;
+
+    fn get_or_create_guid(item: &rss::Item) -> String {
+      if let Some(guid) = item.guid() {
+        return guid.value().to_string();
+      }
+
+      let mut hasher = std::collections::hash_map::DefaultHasher::new();
+      item.link().unwrap_or("").hash(&mut hasher);
+      item.title().unwrap_or("").hash(&mut hasher);
+      item.description().unwrap_or("").hash(&mut hasher);
+
+      format!("gen-{:x}", hasher.finish())
+    }
+
+    for item in rss_channel.items {
+      sqlx::query!(
+        "INSERT OR IGNORE INTO items
+         (channel_id, guid, fetched_at, title, description, content)
+         VALUES (?, ?, ?, ?, ?, ?)",
+        self.id, get_or_create_guid(&item), now.to_rfc2822(),
+        item.title().unwrap_or(""), item.description().unwrap_or(""),
+        item.content().unwrap_or("")
+      )
+      .execute(adapter.get_pool())
+      .await?;
+    }
+
+    Ok(self)
+  }
+}
+
+pub struct Item {
+  id: i64,
 }