1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

use super::Sync;
use anyhow::Result;
use csv::Reader;
use flate2::bufread::GzDecoder;
use openvet_common::rust::CrateName;
use semver::Version;
use serde::Deserialize;
use std::{fs::File, io::BufReader, path::Path};
use tar::Archive;
use tempfile::tempdir;
use tracing::info;

#[derive(Deserialize)]
pub struct CrateRecord {
    id: u64,
    name: CrateName,
    updated_at: String,
    created_at: String,
}

#[derive(Deserialize)]
pub struct VersionRecord {
    id: u64,
    crate_id: u64,
    num: Version,
    updated_at: String,
    created_at: String,
    //yanked: bool,
    crate_size: Option<usize>,
    published_by: Option<u64>,
    checksum: String,
    rust_version: String,
    yank_message: Option<String>,
}

impl Sync {
    pub async fn load_dump(&self, path: &Path) -> Result<()> {
        let dir = tempdir()?;
        info!("Unpacking {path:?} to {}", dir.path().display());
        let file = File::open(path)?;
        let reader = BufReader::new(file);
        let reader = GzDecoder::new(reader);
        let mut archive = Archive::new(reader);
        archive.unpack(dir.path())?;

        //let mut crates = vec![];

        let mut crates_csv = Reader::from_reader(File::open(
            dir.path()
                .join("2024-11-01-020026")
                .join("data")
                .join("crates.csv"),
        )?);
        for record in crates_csv.deserialize() {
            let record: CrateRecord = record?;
        }

        let mut versions_csv = Reader::from_reader(File::open(
            dir.path()
                .join("2024-11-01-020026")
                .join("data")
                .join("versions.csv"),
        )?);
        for record in versions_csv.deserialize() {
            let record: VersionRecord = record?;
        }

        Ok(())
    }
}