Move to DB module for visibility; separate db and client code

koucha/src/fetch.rs

@@ -0,0 +1,87 @@
+use crate::{
+  Result,
+  db::Channel,
+  AdapterClient,
+};
+use reqwest::Url;
+use chrono::{DateTime, Utc};
+use std::hash::{Hash, Hasher};
+
+pub struct FetchedRSSItem {
+  guid: String,
+  title: String,
+  description: String,
+  content: String,
+}
+impl FetchedRSSItem {
+  pub fn guid(&self) -> &str { &self.guid }
+  pub fn title(&self) -> &str { &self.title }
+  pub fn description(&self) -> &str { &self.description }
+  pub fn content(&self) -> &str { &self.content }
+
+  fn parse(item: rss::Item) -> Self {
+    FetchedRSSItem {
+      guid: Self::get_or_create_guid(&item),
+      title: item.title().unwrap_or("").to_string(), 
+      description: item.description().unwrap_or("").to_string(),
+      content: item.content().unwrap_or("").to_string(),
+    }
+  }
+
+  fn get_or_create_guid(item: &rss::Item) -> String {
+    if let Some(guid) = item.guid() {
+      return guid.value().to_string();
+    }
+
+    let mut hasher = std::collections::hash_map::DefaultHasher::new();
+    item.link().unwrap_or("").hash(&mut hasher);
+    item.title().unwrap_or("").hash(&mut hasher);
+    item.description().unwrap_or("").hash(&mut hasher);
+
+    format!("gen-{:x}", hasher.finish())
+  }
+}
+pub struct FetchedRSSChannel {
+  title: String,
+  link: Url,
+  description: String,
+
+  items: Vec<FetchedRSSItem>,
+
+  fetched_at: DateTime<Utc>,
+}
+impl FetchedRSSChannel {
+  pub fn title(&self) -> &str { &self.title }
+  pub fn link(&self) -> &Url { &self.link }
+  pub fn description(&self) -> &str { &self.description }
+  pub fn items(&self) -> &[FetchedRSSItem] { &self.items }
+  pub fn fetched_at(&self) -> &DateTime<Utc> { &self.fetched_at }
+
+  pub async fn fetch_channel(
+    client: &AdapterClient, channel: Channel
+  ) -> Result<Option<Self>> {
+    if channel.should_skip_fetch() {
+      return Ok(None);
+    }
+
+    let bytestream = client.0.get(channel.link().clone())
+      .send().await?
+      .bytes().await?;
+
+    let rss_channel = rss::Channel::read_from(&bytestream[..])?;
+
+    Ok(Some(FetchedRSSChannel::parse(rss_channel)?))
+  }
+
+  fn parse(rss: rss::Channel) -> Result<Self> {
+    Ok(FetchedRSSChannel {
+      title: rss.title,
+      link: Url::parse(&rss.link)?,
+      description: rss.description,
+
+      items: rss.items.into_iter().map(FetchedRSSItem::parse).collect(),
+
+      fetched_at: Utc::now(),
+    })
+  }
+}