gregorbednov/nix-jetbrains-plugins
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #4: make sure plugins are not removed from all_plugins.json, unless unused

Browse source
This commit is contained in:
Marco "Capypara" Köpcke 2024-12-11 12:01:46 +01:00
commit 465f62cc94
No known key found for this signature in database
GPG key ID: 08131EE895D53BDB
612 changed files with 6031 additions and 5477 deletions
Download patch file Download diff file Expand all files Collapse all files

41
generator/src/ides.rs
View file

@ -73,6 +73,29 @@ impl IdeProduct {
}
}
fn try_from_nix_key(code: &str) -> Option<Self> {
Some(match code {
"idea-ultimate" => IdeProduct::IntelliJUltimate,
"idea-community" => IdeProduct::IntelliJCommunity,
"phpstorm" => IdeProduct::PhpStorm,
"webstorm" => IdeProduct::WebStorm,
"pycharm-professional" => IdeProduct::PyCharmProfessional,
"pycharm-community" => IdeProduct::PyCharmCommunity,
"ruby-mine" => IdeProduct::RubyMine,
"clion" => IdeProduct::CLion,
"goland" => IdeProduct::GoLand,
"datagrip" => IdeProduct::DataGrip,
"dataspell" => IdeProduct::DataSpell,
"rider" => IdeProduct::Rider,
"android-studio" => IdeProduct::AndroidStudio,
"rust-rover" => IdeProduct::RustRover,
"aqua" => IdeProduct::Aqua,
"writerside" => IdeProduct::Writerside,
"mps" => IdeProduct::Mps,
_ => return None,
})
}
pub fn nix_key(&self) -> &str {
match self {
IdeProduct::IntelliJUltimate => "idea-ultimate",
@ -103,6 +126,24 @@ pub struct IdeVersion {
pub build_number: String,
}
impl IdeVersion {
/// Create from a JSON filename.
/// WARNING: Does not populate build number!
pub fn from_json_filename(filename: &str) -> Option<Self> {
let filename = filename.strip_suffix(".json")?;
let (product, version) = filename.rsplit_once('-')?;
Some(Self {
ide: IdeProduct::try_from_nix_key(product)?,
version: version.to_string(),
build_number: "".to_string(),
})
}
pub fn to_json_filename(&self) -> String {
format!("{}-{}.json", self.ide.nix_key(), self.version)
}
}
#[derive(Debug, PartialEq, Deserialize)]
pub struct Products {
product: Vec<Product>,

38
generator/src/main.rs
View file

@ -2,7 +2,7 @@ mod ides;
mod logging;
mod plugins;
use clap::Parser;
use clap::{Parser, Subcommand};
use log::info;
use std::path::PathBuf;
use tokio::try_join;
@ -11,7 +11,18 @@ use tokio::try_join;
struct Cli {
#[arg(short, long)]
output_path: PathBuf,
#[clap(subcommand)]
command: Command,
}
#[derive(Subcommand)]
enum Command {
/// Generate the IDE JSON files and create/update all_plugins.json
Generate,
/// Remove all plugins from all_plugins.json that are no longer used in any IDE json file.
Cleanup,
}
const PLUGIN_INDICES: &[&str] = &[
"https://downloads.marketplace.jetbrains.com/files/pluginsXMLIds.json",
"https://downloads.marketplace.jetbrains.com/files/jbPluginsXMLIds.json",
@ -23,6 +34,14 @@ async fn main() -> anyhow::Result<()> {
_ = logging::setup_logging();
info!("Starting...");
match cli.command {
Command::Generate => generate(cli).await,
Command::Cleanup => cleanup(cli).await,
}
}
async fn generate(cli: Cli) -> anyhow::Result<()> {
info!("running generate.");
let (ides, mut plugins, jb_plugins) = try_join!(
ides::collect_ids(),
plugins::index(PLUGIN_INDICES[0]),
@ -38,9 +57,22 @@ async fn main() -> anyhow::Result<()> {
plugins.extend_from_slice(&jb_plugins);
info!("Loading old database.");
let cache_db = plugins::db_cache_load(&cli.output_path).await?;
let mut db = plugins::db_load(&cli.output_path).await?;
info!("Beginning plugin download...");
let db = plugins::build_db(&cache_db, &ides, &plugins).await?;
plugins::db_update(&mut db, &ides, &plugins).await?;
info!("Saving DB...");
plugins::db_save(&cli.output_path, db).await?;
Ok(())
}
async fn cleanup(cli: Cli) -> anyhow::Result<()> {
info!("Loading database and IDE mappings.");
let mut db = plugins::db_load_full(&cli.output_path).await?;
info!("Running cleanup...");
plugins::db_cleanup(&mut db).await?;
info!("Saving DB...");
plugins::db_save(&cli.output_path, db).await?;

162
generator/src/plugins.rs
View file

@ -12,16 +12,18 @@ use std::borrow::Cow;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::fs::exists;
use std::future;
use std::mem::take;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::sync::Arc;
use std::time::Duration;
use tokio::fs::{read_to_string, write};
use tokio::fs::{read_dir, read_to_string, write};
use tokio::process::Command;
use tokio::sync::RwLock;
use tokio::time::timeout;
use tokio_retry2::strategy::ExponentialBackoff;
use tokio_retry2::{Retry, RetryError};
use tokio_stream::wrappers::ReadDirStream;
use version_compare::Version;
use which::which;
@ -41,8 +43,6 @@ impl PluginVersion {
Self(format!("{}{}{}", name, Self::SEPARATOR, version))
}
}
type PluginCache = HashMap<PluginVersion, PluginDbEntry>;
// Plugins for which download requests have 404ed
type FourOFourCache = HashSet<PluginVersion>;
@ -52,6 +52,46 @@ pub struct PluginDb {
ides: HashMap<IdeVersion, BTreeMap<String, String>>,
}
impl PluginDb {
pub fn new() -> Self {
Self {
all_plugins: Default::default(),
ides: Default::default(),
}
}
fn init(init: impl IntoIterator<Item = (PluginVersion, PluginDbEntry)>) -> PluginDb {
Self {
// see insert on why we do this
all_plugins: init
.into_iter()
.map(|(k, v)| {
// see insert on why we do this
let v: &'static _ = Box::leak(Box::new(v));
(k, v)
})
.collect(),
ides: Default::default(),
}
}
pub fn insert(
&mut self,
ideversion: &IdeVersion,
name: &str,
version: &str,
entry: &PluginDbEntry,
) {
let version_entry = self.ides.entry(ideversion.clone()).or_default();
// We leak here since self-referential structs are otherwise a nightmare and it doesn't
// really matter in this CLI app.
self.all_plugins
.entry(PluginVersion::new(name, version))
.or_insert_with(|| Box::leak(Box::new(entry.clone())));
version_entry.insert(name.to_string(), version.to_string());
}
}
#[derive(Debug, PartialEq, Deserialize)]
pub struct PluginDetails {
category: Option<PluginDetailsCategory>,
@ -78,30 +118,6 @@ pub struct PluginDetailsIdeaVersion {
until_build: Option<String>,
}
impl PluginDb {
pub fn new() -> Self {
Self {
all_plugins: Default::default(),
ides: Default::default(),
}
}
pub fn insert(
&mut self,
ideversion: &IdeVersion,
name: &str,
version: &str,
entry: &PluginDbEntry,
) {
let version_entry = self.ides.entry(ideversion.clone()).or_default();
// We leak here since self-referential structs are otherwise a nightmare and it doesn't
// really matter in this CLI app.
self.all_plugins
.entry(PluginVersion::new(name, version))
.or_insert_with(|| Box::leak(Box::new(entry.clone())));
version_entry.insert(name.to_string(), version.to_string());
}
}
#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize)]
pub struct PluginDbEntry {
#[serde(rename = "p")]
@ -114,28 +130,63 @@ pub async fn index(url: &str) -> anyhow::Result<Vec<String>> {
Ok(reqwest::get(url).await?.json().await?)
}
pub async fn db_cache_load(out_dir: &Path) -> anyhow::Result<PluginCache> {
/// Load the plugin database, all_plugins.json only!
pub async fn db_load(out_dir: &Path) -> anyhow::Result<PluginDb> {
let file = out_dir.join(ALL_PLUGINS_JSON);
if exists(&file)? {
Ok(serde_json::from_str(&read_to_string(file).await?)?)
Ok(PluginDb::init(serde_json::from_str::<'_, HashMap<_, _>>(
&read_to_string(file).await?,
)?))
} else {
Ok(PluginCache::new())
Ok(PluginDb::new())
}
}
pub async fn build_db(
cache: &PluginCache,
/// Load the plugin database, including the IDE mappings.
/// WARNING: Does not populate build numbers for IDEs!
pub async fn db_load_full(out_dir: &Path) -> anyhow::Result<PluginDb> {
let mut db = db_load(out_dir).await?;
let db_mut = Arc::new(RwLock::new(&mut db));
ReadDirStream::new(read_dir(out_dir.join("ides")).await?)
.and_then(|file| {
let db_mut = db_mut.clone();
async move {
let Some(ideversion) =
IdeVersion::from_json_filename(&*file.file_name().to_string_lossy())
else {
warn!(
"Invalid JSON file in ide directory skipped: {}",
file.path().display()
);
return Ok(());
};
let ide_mapping: BTreeMap<String, String> =
serde_json::from_str(&read_to_string(file.path()).await?)?;
let mut lck = db_mut.write().await;
let db_mut = &mut *lck;
db_mut.ides.insert(ideversion, ide_mapping);
Ok(())
}
})
.try_collect::<()>()
.await?;
Ok(db)
}
pub async fn db_update(
db: &mut PluginDb,
ides: &[IdeVersion],
pluginkeys: &[String],
) -> anyhow::Result<PluginDb> {
let cache = Arc::new(cache);
) -> anyhow::Result<()> {
let client = Arc::new(
Client::builder()
.timeout(Duration::from_secs(600))
.build()?,
);
let fof_cache = Arc::new(RwLock::new(FourOFourCache::new()));
let db = Arc::new(RwLock::new(PluginDb::new()));
let db = Arc::new(RwLock::new(db));
let mut futures = Vec::new();
@ -143,7 +194,6 @@ pub async fn build_db(
let fof_cache = fof_cache.clone();
let db = db.clone();
let client = client.clone();
let cache = cache.clone();
// Create a future that will be retried 3 times, has a timeout of 1200 seconds per try
// and polls process_plugin to process this plugin for this IDE version. process_plugin
@ -153,7 +203,6 @@ pub async fn build_db(
let fof_cache = fof_cache.clone();
let db = db.clone();
let client = client.clone();
let cache = cache.clone();
async move {
let res = timeout(
Duration::from_secs(1200),
@ -162,7 +211,6 @@ pub async fn build_db(
client.clone(),
ides,
pluginkey,
&cache,
fof_cache.clone(),
),
)
@ -192,7 +240,7 @@ pub async fn build_db(
.try_all(|()| future::ready(true))
.await?;
Ok(Arc::into_inner(db).unwrap().into_inner())
Ok(())
}
/// Various hacks to support (or skip) some very odd cases
@ -209,11 +257,10 @@ fn hacks_for_details_key(pluginkey: &str) -> Option<&str> {
}
async fn process_plugin(
db: Arc<RwLock<PluginDb>>,
db: Arc<RwLock<&mut PluginDb>>,
client: Arc<Client>,
ides: &[IdeVersion],
pluginkey: &str,
cache: &PluginCache,
fof_cache: Arc<RwLock<FourOFourCache>>,
) -> anyhow::Result<()> {
debug!("Processing {pluginkey}...");
@ -252,11 +299,11 @@ async fn process_plugin(
None => warn!("{pluginkey}: IDE {ide:?} not supported."),
Some(version) => {
let entry =
get_db_entry(&client, pluginkey, &version.version, &db, cache, &fof_cache)
.await?;
get_db_entry(&client, pluginkey, &version.version, &db, &fof_cache).await?;
if let Some(entry) = entry {
let mut lck = db.write().await;
lck.insert(ide, pluginkey, &version.version, &entry);
let db_mut = &mut *lck;
db_mut.insert(ide, pluginkey, &version.version, &entry);
}
}
}
@ -289,8 +336,7 @@ async fn get_db_entry<'a>(
client: &Client,
pluginkey: &str,
version: &str,
current_db: &RwLock<PluginDb>,
cache: &'a PluginCache,
current_db: &RwLock<&mut PluginDb>,
fof_cache: &RwLock<FourOFourCache>,
) -> anyhow::Result<Option<Cow<'a, PluginDbEntry>>> {
let key = PluginVersion::new(pluginkey, version);
@ -302,10 +348,6 @@ async fn get_db_entry<'a>(
return Ok(Some(Cow::Borrowed(v)));
}
};
// Look in cache
if let Some(v) = cache.get(&key) {
return Ok(Some(Cow::Borrowed(v)));
}
{
if fof_cache.read().await.contains(&key) {
@ -411,9 +453,27 @@ pub async fn db_save(output_folder: &Path, db: PluginDb) -> anyhow::Result<()> {
// mappings
let output_folder = output_folder.join("ides");
for (ide, plugins) in db.ides {
let out_path = output_folder.join(format!("{}-{}.json", ide.ide.nix_key(), ide.version));
let out_path = output_folder.join(ide.to_json_filename());
debug!("Generating {out_path:?}...");
write(out_path, serde_json::to_string_pretty(&plugins)?).await?;
}
Ok(())
}
pub async fn db_cleanup(db: &mut PluginDb) -> anyhow::Result<()> {
let mut used_keys: HashSet<_> = db
.ides
.values()
.flat_map(|ides| {
ides.iter()
.map(|(name, version)| PluginVersion::new(name, version))
})
.collect();
db.all_plugins = take(&mut db.all_plugins)
.into_iter()
.filter(|(k, _)| used_keys.contains(k))
.collect();
Ok(())
}