Add web crawler infrastructure and HangarMC implementation

This commit is contained in:
Rbanh committed 2025-03-29 01:00:49 -04:00
1 parent d9cf404402
commit eb51afaea8
7 files changed
+1466 -32

No files matched your search

+203
View File
@@ -0,0 +1,203 @@
use std::error::Error;
use std::path::Path;
use serde::{Serialize, Deserialize};
use crate::{HttpClient, RepositoryCrawler, RepositoryPlugin, RepositorySource};
// HangarMC API response structures
#[derive(Debug, Serialize, Deserialize)]
struct HangarProjectsResponse {
pagination: HangarPagination,
result: Vec<HangarProject>,
}
#[derive(Debug, Serialize, Deserialize)]
struct HangarPagination {
limit: u32,
offset: u32,
count: u32,
}
#[derive(Debug, Serialize, Deserialize)]
struct HangarProject {
id: String,
name: String,
namespace: HangarNamespace,
category: String,
description: Option<String>,
stats: HangarStats,
last_updated: String,
icon_url: Option<String>,
created_at: String,
visibility: String,
}
#[derive(Debug, Serialize, Deserialize)]
struct HangarNamespace {
owner: String,
slug: String,
}
#[derive(Debug, Serialize, Deserialize)]
struct HangarStats {
views: u64,
downloads: u64,
recent_views: u64,
recent_downloads: u64,
stars: u64,
watchers: u64,
}
#[derive(Debug, Serialize, Deserialize)]
struct HangarVersionsResponse {
pagination: HangarPagination,
result: Vec<HangarVersion>,
}
#[derive(Debug, Serialize, Deserialize)]
struct HangarVersion {
name: String,
created_at: String,
description: Option<String>,
downloads: u64,
file_size: u64,
platform_versions: Vec<String>,
}
// HangarMC crawler implementation
pub struct HangarCrawler {
client: HttpClient,
api_base_url: String,
}
impl HangarCrawler {
pub fn new() -> Self {
HangarCrawler {
client: HttpClient::new(),
api_base_url: "https://hangar.papermc.io/api/v1".to_string(),
}
}
fn get_project_details(&self, owner: &str, slug: &str) -> Result<HangarProject, Box<dyn Error>> {
let url = format!("{}/projects/{}/{}", self.api_base_url, owner, slug);
let response = self.client.get(&url)?;
let project: HangarProject = serde_json::from_str(&response)?;
Ok(project)
}
fn get_project_versions(&self, owner: &str, slug: &str) -> Result<Vec<HangarVersion>, Box<dyn Error>> {
let url = format!("{}/projects/{}/{}/versions", self.api_base_url, owner, slug);
let response = self.client.get(&url)?;
let versions_response: HangarVersionsResponse = serde_json::from_str(&response)?;
Ok(versions_response.result)
}
fn build_download_url(&self, owner: &str, slug: &str, version: &str) -> String {
format!("https://hangar.papermc.io/api/v1/projects/{}/{}/versions/{}/download", owner, slug, version)
}
}
impl RepositoryCrawler for HangarCrawler {
fn search(&self, query: &str) -> Result<Vec<RepositoryPlugin>, Box<dyn Error>> {
let url = format!("{}/projects?query={}&limit=20", self.api_base_url, query);
let response = self.client.get(&url)?;
let projects_response: HangarProjectsResponse = serde_json::from_str(&response)?;
let mut results = Vec::new();
for project in projects_response.result {
// For each project, get the latest version
let versions = self.get_project_versions(&project.namespace.owner, &project.namespace.slug)?;
if let Some(latest_version) = versions.first() {
results.push(RepositoryPlugin {
id: format!("{}/{}", project.namespace.owner, project.namespace.slug),
name: project.name,
version: latest_version.name.clone(),
description: project.description,
authors: vec![project.namespace.owner.clone()],
download_url: self.build_download_url(&project.namespace.owner, &project.namespace.slug, &latest_version.name),
repository: RepositorySource::HangarMC,
page_url: format!("https://hangar.papermc.io/{}/{}", project.namespace.owner, project.namespace.slug),
download_count: Some(project.stats.downloads),
last_updated: Some(project.last_updated),
icon_url: project.icon_url,
minecraft_versions: latest_version.platform_versions.clone(),
categories: vec![project.category],
rating: None, // HangarMC uses stars, not ratings
file_size: Some(latest_version.file_size),
file_hash: None, // HangarMC API doesn't provide file hashes
});
}
}
Ok(results)
}
fn get_plugin_details(&self, plugin_id: &str) -> Result<RepositoryPlugin, Box<dyn Error>> {
let parts: Vec<&str> = plugin_id.split('/').collect();
if parts.len() != 2 {
return Err("Invalid plugin ID format for HangarMC. Expected 'owner/slug'".into());
}
let owner = parts[0];
let slug = parts[1];
let project = self.get_project_details(owner, slug)?;
let versions = self.get_project_versions(owner, slug)?;
if let Some(latest_version) = versions.first() {
Ok(RepositoryPlugin {
id: plugin_id.to_string(),
name: project.name,
version: latest_version.name.clone(),
description: project.description,
authors: vec![project.namespace.owner.clone()],
download_url: self.build_download_url(owner, slug, &latest_version.name),
repository: RepositorySource::HangarMC,
page_url: format!("https://hangar.papermc.io/{}/{}", owner, slug),
download_count: Some(project.stats.downloads),
last_updated: Some(project.last_updated),
icon_url: project.icon_url,
minecraft_versions: latest_version.platform_versions.clone(),
categories: vec![project.category],
rating: None,
file_size: Some(latest_version.file_size),
file_hash: None,
})
} else {
Err("No versions found for this plugin".into())
}
}
fn get_plugin_versions(&self, plugin_id: &str) -> Result<Vec<String>, Box<dyn Error>> {
let parts: Vec<&str> = plugin_id.split('/').collect();
if parts.len() != 2 {
return Err("Invalid plugin ID format for HangarMC. Expected 'owner/slug'".into());
}
let owner = parts[0];
let slug = parts[1];
let versions = self.get_project_versions(owner, slug)?;
Ok(versions.into_iter().map(|v| v.name).collect())
}
fn download_plugin(&self, plugin_id: &str, version: &str, destination: &Path) -> Result<String, Box<dyn Error>> {
let parts: Vec<&str> = plugin_id.split('/').collect();
if parts.len() != 2 {
return Err("Invalid plugin ID format for HangarMC. Expected 'owner/slug'".into());
}
let owner = parts[0];
let slug = parts[1];
let download_url = self.build_download_url(owner, slug, version);
self.client.download(&download_url, destination)?;
Ok(destination.to_string_lossy().to_string())
}
fn get_repository_name(&self) -> RepositorySource {
RepositorySource::HangarMC
}
}
+4
View File
@@ -0,0 +1,4 @@
pub mod hangar;
// Re-export the crawler implementations
pub use hangar::HangarCrawler;