From 19c1578320ef98d76c3e758316d8774ac51a47fe Mon Sep 17 00:00:00 2001 From: edef Date: Tue, 10 May 2022 01:32:17 +0000 Subject: ripple/fossil: permit importing trees from git repos Change-Id: I8329f14bddaaa2d141e82c087821e4602bd1259a --- ripple/Cargo.lock | 168 +++++++++++++++++++++++++++++++++++++++++++++++ ripple/fossil/Cargo.toml | 1 + ripple/fossil/src/lib.rs | 57 ++++++++++++++++ ripple/shell.nix | 3 + 4 files changed, 229 insertions(+) diff --git a/ripple/Cargo.lock b/ripple/Cargo.lock index c5aae4b..5b65622 100644 --- a/ripple/Cargo.lock +++ b/ripple/Cargo.lock @@ -131,6 +131,9 @@ name = "cc" version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2" +dependencies = [ + "jobserver", +] [[package]] name = "cfg-if" @@ -351,6 +354,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + [[package]] name = "fossil" version = "0.1.0" @@ -364,6 +377,7 @@ dependencies = [ "criterion", "env_logger", "fuser", + "git2", "lazy_static", "libc", "log", @@ -429,6 +443,21 @@ dependencies = [ "wasi", ] +[[package]] +name = "git2" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e77a14ffc6ba4ad5188d6cf428894c4fcfda725326b37558f35bb677e712cec" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + [[package]] name = "half" version = "1.8.2" @@ -471,6 +500,17 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indexmap" version = "1.7.0" @@ -511,6 +551,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +[[package]] +name = "jobserver" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.57" @@ -532,6 +581,46 @@ version = "0.2.112" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125" +[[package]] +name = "libgit2-sys" +version = "0.13.3+1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c24d36c3ac9b9996a2418d6bf428cc0bc5d1a814a84303fc60986088c5ed60de" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + +[[package]] +name = "libssh2-sys" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b094a36eb4b8b8c8a7b4b8ae43b2944502be3e59cd87687595cf6b0a71b3f4ca" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e7e15d7610cce1d9752e137625f14e61a28cd45929b6e12e47b50fe154ee2e" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "lock_api" version = "0.4.4" @@ -550,6 +639,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + [[package]] name = "memchr" version = "2.4.1" @@ -618,6 +713,25 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5fd19fb3e0a8191c1e34935718976a3e70c112ab9a24af6d7cadccd9d90bc0" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "os_str_bytes" version = "6.0.0" @@ -659,6 +773,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + [[package]] name = "petgraph" version = "0.5.1" @@ -1091,12 +1211,42 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + [[package]] name = "typenum" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.8.0" @@ -1115,6 +1265,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + [[package]] name = "users" version = "0.11.0" @@ -1125,6 +1287,12 @@ dependencies = [ "log", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.3" diff --git a/ripple/fossil/Cargo.toml b/ripple/fossil/Cargo.toml index 2ff2288..663919f 100644 --- a/ripple/fossil/Cargo.toml +++ b/ripple/fossil/Cargo.toml @@ -15,6 +15,7 @@ bytes = "1.0.1" clap = { version = "3.1.15", features = ["derive"] } env_logger = "0.9.0" fuser = "0.11.0" +git2 = "0.14.3" lazy_static = "1.4.0" libc = "0.2.112" log = "0.4.14" diff --git a/ripple/fossil/src/lib.rs b/ripple/fossil/src/lib.rs index 007a771..b3a1c3d 100644 --- a/ripple/fossil/src/lib.rs +++ b/ripple/fossil/src/lib.rs @@ -14,6 +14,7 @@ use { io::{self, Read, Write}, os::unix::{fs::PermissionsExt, prelude::FileExt}, path::Path, + str, }, }; @@ -74,6 +75,62 @@ impl Store { self.meta.flush().unwrap(); } + pub fn add_git_tree(&self, repo: &git2::Repository, tree: git2::Oid) -> DirectoryRef { + let tree = repo.find_tree(tree).unwrap(); + let d = self.add_git_tree_inner(repo, &tree); + self.flush(); + d + } + + fn add_git_tree_inner(&self, repo: &git2::Repository, tree: &git2::Tree) -> DirectoryRef { + let mut d = Directory::new(); + let mut size: u32 = 0; + + for entry in tree { + let name = entry.name().expect("invalid UTF-8"); + let object = entry.to_object(repo).unwrap(); + let kind = entry.kind().unwrap(); + let mode = entry.filemode(); + + let child = match kind { + git2::ObjectType::Tree => { + let tree = object.as_tree().unwrap(); + Node::Directory(self.add_git_tree_inner(repo, tree)) + } + git2::ObjectType::Blob if mode == 0o120000 => { + let target = object.as_blob().unwrap().content(); + let target = str::from_utf8(target).unwrap(); + Node::Link { + target: target.to_owned(), + } + } + git2::ObjectType::Blob => { + let executable = match mode { + 0o100644 => true, + 0o100755 => false, + _ => panic!("unexpected mode: {:o} for file {:?}", mode, name), + }; + + let data = object.as_blob().unwrap().content(); + Node::File(FileRef { + ident: self.write_blob(data), + size: data.len() as u32, + executable, + }) + } + kind => panic!("unexpected object kind: {:?}", kind), + }; + + size = size.checked_add(child.size()).expect("overflow"); + d.children.insert(name.to_owned(), child); + } + + let blob = d.into_pb().encode_to_vec(); + let ident = self.write_blob(&blob); + + DirectoryRef { ident, size } + } + pub fn add_directory(&self, path: impl AsRef) -> DirectoryRef { let d = self.add_directory_inner(path.as_ref()); self.flush(); diff --git a/ripple/shell.nix b/ripple/shell.nix index 96ed84f..cc9c44e 100644 --- a/ripple/shell.nix +++ b/ripple/shell.nix @@ -22,6 +22,9 @@ mkShell { # needed by fuser pkgconfig fuse + + # needed by git2 + openssl ]; # needed by prost-build -- cgit 1.4.1