Fetching data with channels
This commit is contained in:
parent
f42e558db9
commit
22871f5789
SSH key fingerprint:
SHA256:5DJcfxa5/fKCYn57dcabJa2vN2e6eT0pBerYi5SUbto
3 changed files with 225 additions and 37 deletions
73
koucha/Cargo.lock
generated
73
koucha/Cargo.lock
generated
|
|
@ -8,6 +8,15 @@ version = "0.2.21"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "atoi"
|
||||
version = "2.0.0"
|
||||
|
|
@ -200,7 +209,11 @@ version = "0.4.43"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118"
|
||||
dependencies = [
|
||||
"iana-time-zone",
|
||||
"js-sys",
|
||||
"num-traits",
|
||||
"wasm-bindgen",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -836,6 +849,30 @@ dependencies = [
|
|||
"windows-registry",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.64"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb"
|
||||
dependencies = [
|
||||
"android_system_properties",
|
||||
"core-foundation-sys",
|
||||
"iana-time-zone-haiku",
|
||||
"js-sys",
|
||||
"log",
|
||||
"wasm-bindgen",
|
||||
"windows-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone-haiku"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_collections"
|
||||
version = "2.1.1"
|
||||
|
|
@ -1023,6 +1060,7 @@ name = "koucha"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"axum",
|
||||
"chrono",
|
||||
"reqwest",
|
||||
"rss",
|
||||
"sqlx",
|
||||
|
|
@ -2547,6 +2585,41 @@ dependencies = [
|
|||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.62.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
|
||||
dependencies = [
|
||||
"windows-implement",
|
||||
"windows-interface",
|
||||
"windows-link",
|
||||
"windows-result",
|
||||
"windows-strings",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-implement"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-interface"
|
||||
version = "0.59.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
|
|
|
|||
|
|
@ -9,3 +9,4 @@ reqwest = "0.13.1"
|
|||
rss = "2.0.12"
|
||||
tokio = { version = "1.49.0", features = ["full"] }
|
||||
sqlx = { version = "0.8.6", features = [ "runtime-tokio", "sqlite" ] }
|
||||
chrono = "0.4.43"
|
||||
|
|
|
|||
|
|
@ -1,9 +1,12 @@
|
|||
use std::error::Error;
|
||||
use reqwest::{
|
||||
IntoUrl,
|
||||
Client,
|
||||
use std::{
|
||||
error::Error,
|
||||
hash::{Hash, Hasher},
|
||||
};
|
||||
use reqwest::Url;
|
||||
use chrono::{
|
||||
Utc,
|
||||
DateTime,
|
||||
};
|
||||
use rss::Channel as RawChannel;
|
||||
|
||||
type Result<T> = std::result::Result<T, Box<dyn Error>>;
|
||||
|
||||
|
|
@ -26,33 +29,42 @@ impl AdapterOptions {
|
|||
pub async fn create(self) -> Result<Adapter> {
|
||||
let db = sqlx::sqlite::SqlitePoolOptions::new()
|
||||
.connect(&self.database_url).await?;
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
Ok(Adapter { db })
|
||||
Ok(Adapter { db, client })
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Adapter {
|
||||
db: sqlx::SqlitePool,
|
||||
client: reqwest::Client,
|
||||
}
|
||||
|
||||
impl Adapter {
|
||||
pub async fn get_all_users(&self) -> Result<Vec<User>> {
|
||||
let users_query = sqlx::query!("SELECT id, name FROM users")
|
||||
.fetch_all(&self.db).await?;
|
||||
let users = sqlx::query_as!(
|
||||
User,
|
||||
"SELECT id, name FROM users"
|
||||
).fetch_all(&self.db).await?;
|
||||
|
||||
let mut all_users: Vec<User> = Vec::with_capacity(users_query.len());
|
||||
|
||||
for user in users_query {
|
||||
all_users.push(User {
|
||||
id: user.id,
|
||||
name: user.name,
|
||||
})
|
||||
}
|
||||
|
||||
Ok(all_users)
|
||||
Ok(users)
|
||||
}
|
||||
|
||||
// pub async fn update_channels(&self) -> Result<()> {
|
||||
//
|
||||
// }
|
||||
//
|
||||
// async fn get_all_channels(&self) -> Result<Vec<impl Channel>> {
|
||||
// let users = sqlx::query_as!(
|
||||
// Channel,
|
||||
// "SELECT id FROM channels"
|
||||
// ).fetch_all(&self.db).await?;
|
||||
//
|
||||
// Ok(users)
|
||||
// }
|
||||
|
||||
fn get_pool(&self) -> &sqlx::SqlitePool { &self.db }
|
||||
fn get_client(&self) -> &reqwest::client { &self.client }
|
||||
}
|
||||
|
||||
pub struct User {
|
||||
|
|
@ -61,15 +73,15 @@ pub struct User {
|
|||
}
|
||||
|
||||
impl User {
|
||||
pub async fn get_by_id(adapter: &Adapter, id: i64) -> Result<Self> {
|
||||
let user = sqlx::query!("SELECT name FROM users WHERE id = ?", id)
|
||||
.fetch_one(adapter.get_pool()).await?;
|
||||
|
||||
Ok(Self {
|
||||
id: id,
|
||||
name: user.name,
|
||||
})
|
||||
}
|
||||
// async fn get_by_id(adapter: &Adapter, id: i64) -> Result<Self> {
|
||||
// let user = sqlx::query!("SELECT name FROM users WHERE id = ?", id)
|
||||
// .fetch_one(adapter.get_pool()).await?;
|
||||
//
|
||||
// Ok(Self {
|
||||
// id: id,
|
||||
// name: user.name,
|
||||
// })
|
||||
// }
|
||||
|
||||
pub async fn create(adapter: &Adapter, name: &str) -> Result<Self> {
|
||||
let result = sqlx::query!("INSERT INTO users (name) VALUES (?)", name)
|
||||
|
|
@ -93,21 +105,123 @@ impl User {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_feeds(&self, adapter: &Adapter) -> Result<Vec<Feed>> {
|
||||
let feeds = sqlx::query_as!(
|
||||
Feed,
|
||||
"SELECT id FROM feeds WHERE user_id = ?",
|
||||
self.id
|
||||
).fetch_all(adapter.get_pool()).await?;
|
||||
|
||||
Ok(feeds)
|
||||
}
|
||||
|
||||
pub fn name(&self) -> &str { &self.name }
|
||||
pub fn id(&self) -> i64 { self.id }
|
||||
}
|
||||
|
||||
pub struct Feed {
|
||||
id: i64,
|
||||
}
|
||||
|
||||
impl Feed {
|
||||
pub async fn get_items(
|
||||
&self, adapter: &Adapter, limit: u8, offset: u32) -> Result<Vec<Item>> {
|
||||
let items = sqlx::query_as!(
|
||||
Item,
|
||||
"SELECT item_id as id FROM feed_items
|
||||
WHERE feed_id = ? AND archived = FALSE
|
||||
ORDER BY score DESC
|
||||
LIMIT ? OFFSET ?",
|
||||
self.id, limit, offset
|
||||
).fetch_all(adapter.get_pool()).await?;
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
|
||||
pub async fn get_channels(&self, adapter: &Adapter) -> Result<Vec<Channel>> {
|
||||
let db_channels = sqlx::query!(
|
||||
"SELECT c.id as `id!`, c.title, c.link, c.description, c.last_fetched
|
||||
FROM channels c
|
||||
JOIN feed_channels fc on c.id = fc.channel_id
|
||||
WHERE fc.feed_id = ?",
|
||||
self.id
|
||||
).fetch_all(adapter.get_pool()).await?;
|
||||
let mut channels = Vec::with_capacity(db_channels.len());
|
||||
for db_channel in db_channels {
|
||||
channels.push(Channel {
|
||||
id: db_channel.id,
|
||||
title: db_channel.title,
|
||||
link: Url::parse(&db_channel.link)?,
|
||||
description: db_channel.description,
|
||||
last_fetched: db_channel.last_fetched.as_deref()
|
||||
.map(DateTime::parse_from_rfc2822)
|
||||
.transpose()?
|
||||
.map(|dt| dt.with_timezone(&Utc)),
|
||||
})
|
||||
}
|
||||
Ok(channels)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Channel {
|
||||
pub channel: rss::Channel,
|
||||
id: i64,
|
||||
title: String,
|
||||
link: Url,
|
||||
description: Option<String>,
|
||||
last_fetched: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
pub async fn fetch_channel<T: IntoUrl>(
|
||||
client: &Client, url: T) -> Result<Channel> {
|
||||
let content = client.get(url)
|
||||
.send().await?
|
||||
.bytes().await?;
|
||||
impl Channel {
|
||||
pub async fn fetch(mut self, adapter: &Adapter) -> Result<Self> {
|
||||
let bytestream = adapter.get_client().get(self.link.clone())
|
||||
.send().await?
|
||||
.bytes().await?;
|
||||
|
||||
let raw_channel = RawChannel::read_from(&content[..])?;
|
||||
println!("{}", raw_channel.title);
|
||||
Ok(Channel { channel: raw_channel })
|
||||
let rss_channel = rss::Channel::read_from(&bytestream[..])?;
|
||||
self.title = rss_channel.title;
|
||||
self.link = Url::parse(&rss_channel.link)?;
|
||||
self.description = Some(rss_channel.description);
|
||||
let now = Utc::now();
|
||||
self.last_fetched = Some(now);
|
||||
|
||||
sqlx::query!(
|
||||
"UPDATE channels
|
||||
SET title = ?, link = ?, description = ?,
|
||||
last_fetched = ?
|
||||
WHERE id = ?",
|
||||
self.title, self.link.as_str(), self.description, now.to_rfc2822(),
|
||||
self.id
|
||||
).execute(adapter.get_pool()).await?;
|
||||
|
||||
fn get_or_create_guid(item: &rss::Item) -> String {
|
||||
if let Some(guid) = item.guid() {
|
||||
return guid.value().to_string();
|
||||
}
|
||||
|
||||
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
||||
item.link().unwrap_or("").hash(&mut hasher);
|
||||
item.title().unwrap_or("").hash(&mut hasher);
|
||||
item.description().unwrap_or("").hash(&mut hasher);
|
||||
|
||||
format!("gen-{:x}", hasher.finish())
|
||||
}
|
||||
|
||||
for item in rss_channel.items {
|
||||
sqlx::query!(
|
||||
"INSERT OR IGNORE INTO items
|
||||
(channel_id, guid, fetched_at, title, description, content)
|
||||
VALUES (?, ?, ?, ?, ?, ?)",
|
||||
self.id, get_or_create_guid(&item), now.to_rfc2822(),
|
||||
item.title().unwrap_or(""), item.description().unwrap_or(""),
|
||||
item.content().unwrap_or("")
|
||||
)
|
||||
.execute(adapter.get_pool())
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Item {
|
||||
id: i64,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue