Add web crawler infrastructure and HangarMC implementation
This commit is contained in:
1 parent
d9cf404402
commit
eb51afaea8
7 files changed
+1466
-32
No files matched your search
@@ -0,0 +1,203 @@
|
||||
use std::error::Error;
|
||||
use std::path::Path;
|
||||
use serde::{Serialize, Deserialize};
|
||||
use crate::{HttpClient, RepositoryCrawler, RepositoryPlugin, RepositorySource};
|
||||
|
||||
// HangarMC API response structures
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct HangarProjectsResponse {
|
||||
pagination: HangarPagination,
|
||||
result: Vec<HangarProject>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct HangarPagination {
|
||||
limit: u32,
|
||||
offset: u32,
|
||||
count: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct HangarProject {
|
||||
id: String,
|
||||
name: String,
|
||||
namespace: HangarNamespace,
|
||||
category: String,
|
||||
description: Option<String>,
|
||||
stats: HangarStats,
|
||||
last_updated: String,
|
||||
icon_url: Option<String>,
|
||||
created_at: String,
|
||||
visibility: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct HangarNamespace {
|
||||
owner: String,
|
||||
slug: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct HangarStats {
|
||||
views: u64,
|
||||
downloads: u64,
|
||||
recent_views: u64,
|
||||
recent_downloads: u64,
|
||||
stars: u64,
|
||||
watchers: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct HangarVersionsResponse {
|
||||
pagination: HangarPagination,
|
||||
result: Vec<HangarVersion>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct HangarVersion {
|
||||
name: String,
|
||||
created_at: String,
|
||||
description: Option<String>,
|
||||
downloads: u64,
|
||||
file_size: u64,
|
||||
platform_versions: Vec<String>,
|
||||
}
|
||||
|
||||
// HangarMC crawler implementation
|
||||
pub struct HangarCrawler {
|
||||
client: HttpClient,
|
||||
api_base_url: String,
|
||||
}
|
||||
|
||||
impl HangarCrawler {
|
||||
pub fn new() -> Self {
|
||||
HangarCrawler {
|
||||
client: HttpClient::new(),
|
||||
api_base_url: "https://hangar.papermc.io/api/v1".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_project_details(&self, owner: &str, slug: &str) -> Result<HangarProject, Box<dyn Error>> {
|
||||
let url = format!("{}/projects/{}/{}", self.api_base_url, owner, slug);
|
||||
let response = self.client.get(&url)?;
|
||||
let project: HangarProject = serde_json::from_str(&response)?;
|
||||
Ok(project)
|
||||
}
|
||||
|
||||
fn get_project_versions(&self, owner: &str, slug: &str) -> Result<Vec<HangarVersion>, Box<dyn Error>> {
|
||||
let url = format!("{}/projects/{}/{}/versions", self.api_base_url, owner, slug);
|
||||
let response = self.client.get(&url)?;
|
||||
let versions_response: HangarVersionsResponse = serde_json::from_str(&response)?;
|
||||
Ok(versions_response.result)
|
||||
}
|
||||
|
||||
fn build_download_url(&self, owner: &str, slug: &str, version: &str) -> String {
|
||||
format!("https://hangar.papermc.io/api/v1/projects/{}/{}/versions/{}/download", owner, slug, version)
|
||||
}
|
||||
}
|
||||
|
||||
impl RepositoryCrawler for HangarCrawler {
|
||||
fn search(&self, query: &str) -> Result<Vec<RepositoryPlugin>, Box<dyn Error>> {
|
||||
let url = format!("{}/projects?query={}&limit=20", self.api_base_url, query);
|
||||
let response = self.client.get(&url)?;
|
||||
|
||||
let projects_response: HangarProjectsResponse = serde_json::from_str(&response)?;
|
||||
let mut results = Vec::new();
|
||||
|
||||
for project in projects_response.result {
|
||||
// For each project, get the latest version
|
||||
let versions = self.get_project_versions(&project.namespace.owner, &project.namespace.slug)?;
|
||||
|
||||
if let Some(latest_version) = versions.first() {
|
||||
results.push(RepositoryPlugin {
|
||||
id: format!("{}/{}", project.namespace.owner, project.namespace.slug),
|
||||
name: project.name,
|
||||
version: latest_version.name.clone(),
|
||||
description: project.description,
|
||||
authors: vec![project.namespace.owner.clone()],
|
||||
download_url: self.build_download_url(&project.namespace.owner, &project.namespace.slug, &latest_version.name),
|
||||
repository: RepositorySource::HangarMC,
|
||||
page_url: format!("https://hangar.papermc.io/{}/{}", project.namespace.owner, project.namespace.slug),
|
||||
download_count: Some(project.stats.downloads),
|
||||
last_updated: Some(project.last_updated),
|
||||
icon_url: project.icon_url,
|
||||
minecraft_versions: latest_version.platform_versions.clone(),
|
||||
categories: vec![project.category],
|
||||
rating: None, // HangarMC uses stars, not ratings
|
||||
file_size: Some(latest_version.file_size),
|
||||
file_hash: None, // HangarMC API doesn't provide file hashes
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
fn get_plugin_details(&self, plugin_id: &str) -> Result<RepositoryPlugin, Box<dyn Error>> {
|
||||
let parts: Vec<&str> = plugin_id.split('/').collect();
|
||||
if parts.len() != 2 {
|
||||
return Err("Invalid plugin ID format for HangarMC. Expected 'owner/slug'".into());
|
||||
}
|
||||
|
||||
let owner = parts[0];
|
||||
let slug = parts[1];
|
||||
|
||||
let project = self.get_project_details(owner, slug)?;
|
||||
let versions = self.get_project_versions(owner, slug)?;
|
||||
|
||||
if let Some(latest_version) = versions.first() {
|
||||
Ok(RepositoryPlugin {
|
||||
id: plugin_id.to_string(),
|
||||
name: project.name,
|
||||
version: latest_version.name.clone(),
|
||||
description: project.description,
|
||||
authors: vec![project.namespace.owner.clone()],
|
||||
download_url: self.build_download_url(owner, slug, &latest_version.name),
|
||||
repository: RepositorySource::HangarMC,
|
||||
page_url: format!("https://hangar.papermc.io/{}/{}", owner, slug),
|
||||
download_count: Some(project.stats.downloads),
|
||||
last_updated: Some(project.last_updated),
|
||||
icon_url: project.icon_url,
|
||||
minecraft_versions: latest_version.platform_versions.clone(),
|
||||
categories: vec![project.category],
|
||||
rating: None,
|
||||
file_size: Some(latest_version.file_size),
|
||||
file_hash: None,
|
||||
})
|
||||
} else {
|
||||
Err("No versions found for this plugin".into())
|
||||
}
|
||||
}
|
||||
|
||||
fn get_plugin_versions(&self, plugin_id: &str) -> Result<Vec<String>, Box<dyn Error>> {
|
||||
let parts: Vec<&str> = plugin_id.split('/').collect();
|
||||
if parts.len() != 2 {
|
||||
return Err("Invalid plugin ID format for HangarMC. Expected 'owner/slug'".into());
|
||||
}
|
||||
|
||||
let owner = parts[0];
|
||||
let slug = parts[1];
|
||||
|
||||
let versions = self.get_project_versions(owner, slug)?;
|
||||
Ok(versions.into_iter().map(|v| v.name).collect())
|
||||
}
|
||||
|
||||
fn download_plugin(&self, plugin_id: &str, version: &str, destination: &Path) -> Result<String, Box<dyn Error>> {
|
||||
let parts: Vec<&str> = plugin_id.split('/').collect();
|
||||
if parts.len() != 2 {
|
||||
return Err("Invalid plugin ID format for HangarMC. Expected 'owner/slug'".into());
|
||||
}
|
||||
|
||||
let owner = parts[0];
|
||||
let slug = parts[1];
|
||||
|
||||
let download_url = self.build_download_url(owner, slug, version);
|
||||
self.client.download(&download_url, destination)?;
|
||||
|
||||
Ok(destination.to_string_lossy().to_string())
|
||||
}
|
||||
|
||||
fn get_repository_name(&self) -> RepositorySource {
|
||||
RepositorySource::HangarMC
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
pub mod hangar;
|
||||
|
||||
// Re-export the crawler implementations
|
||||
pub use hangar::HangarCrawler;
|
||||
Reference in new issue
Block a user