2017-07-21 09:21:59 +00:00
|
|
|
use prelude::*;
|
2017-03-21 10:28:11 +00:00
|
|
|
|
2017-03-25 11:43:49 +00:00
|
|
|
use std::collections::HashSet;
|
2017-03-20 13:03:29 +00:00
|
|
|
|
|
|
|
|
|
|
|
impl Repository {
|
|
|
|
fn delete_bundle(&mut self, id: u32) -> Result<(), RepositoryError> {
|
|
|
|
if let Some(bundle) = self.bundle_map.remove(id) {
|
2017-03-22 11:27:17 +00:00
|
|
|
try!(self.bundles.delete_bundle(&bundle));
|
2017-03-20 13:03:29 +00:00
|
|
|
Ok(())
|
|
|
|
} else {
|
2017-04-10 15:53:26 +00:00
|
|
|
Err(IntegrityError::MissingBundleId(id).into())
|
2017-03-20 13:03:29 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-07-21 09:21:59 +00:00
|
|
|
pub fn vacuum(
|
|
|
|
&mut self,
|
|
|
|
ratio: f32,
|
|
|
|
combine: bool,
|
|
|
|
force: bool,
|
|
|
|
) -> Result<(), RepositoryError> {
|
2017-03-20 13:03:29 +00:00
|
|
|
try!(self.flush());
|
2017-03-24 07:56:57 +00:00
|
|
|
info!("Locking repository");
|
2017-04-12 06:32:27 +00:00
|
|
|
try!(self.write_mode());
|
2017-03-24 07:56:57 +00:00
|
|
|
let _lock = try!(self.lock(true));
|
2017-04-17 09:04:36 +00:00
|
|
|
// analyze_usage will set the dirty flag
|
2017-03-20 13:03:29 +00:00
|
|
|
info!("Analyzing chunk usage");
|
2017-03-22 08:19:16 +00:00
|
|
|
let usage = try!(self.analyze_usage());
|
2017-03-25 11:43:49 +00:00
|
|
|
let mut data_total = 0;
|
|
|
|
let mut data_used = 0;
|
|
|
|
for bundle in usage.values() {
|
|
|
|
data_total += bundle.info.encoded_size;
|
|
|
|
data_used += bundle.get_used_size();
|
|
|
|
}
|
2017-07-21 09:21:59 +00:00
|
|
|
info!(
|
|
|
|
"Usage: {} of {}, {:.1}%",
|
|
|
|
to_file_size(data_used as u64),
|
|
|
|
to_file_size(data_total as u64),
|
|
|
|
data_used as f32 / data_total as f32 * 100.0
|
|
|
|
);
|
2017-03-20 13:03:29 +00:00
|
|
|
let mut rewrite_bundles = HashSet::new();
|
|
|
|
let mut reclaim_space = 0;
|
2018-02-19 20:18:47 +00:00
|
|
|
let mut rewrite_data = 0;
|
2017-03-20 13:03:29 +00:00
|
|
|
for (id, bundle) in &usage {
|
2017-03-25 11:43:49 +00:00
|
|
|
if bundle.get_usage_ratio() <= ratio {
|
2017-03-20 13:03:29 +00:00
|
|
|
rewrite_bundles.insert(*id);
|
2017-03-25 11:43:49 +00:00
|
|
|
reclaim_space += bundle.get_unused_size();
|
2018-02-19 20:18:47 +00:00
|
|
|
rewrite_data += bundle.get_used_size();
|
2017-03-20 13:03:29 +00:00
|
|
|
}
|
|
|
|
}
|
2017-04-17 14:31:58 +00:00
|
|
|
if combine {
|
|
|
|
let mut small_meta = vec![];
|
|
|
|
let mut small_data = vec![];
|
|
|
|
for (id, bundle) in &usage {
|
|
|
|
if bundle.info.encoded_size * 4 < self.config.bundle_size {
|
|
|
|
match bundle.info.mode {
|
|
|
|
BundleMode::Meta => small_meta.push(*id),
|
|
|
|
BundleMode::Data => small_data.push(*id),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if small_meta.len() >= 2 {
|
|
|
|
for bundle in small_meta {
|
|
|
|
rewrite_bundles.insert(bundle);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if small_data.len() >= 2 {
|
|
|
|
for bundle in small_data {
|
|
|
|
rewrite_bundles.insert(bundle);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2017-07-21 09:21:59 +00:00
|
|
|
info!(
|
2018-02-19 20:18:47 +00:00
|
|
|
"Reclaiming about {} by rewriting {} bundles ({})",
|
2017-07-21 09:21:59 +00:00
|
|
|
to_file_size(reclaim_space as u64),
|
2018-02-19 20:18:47 +00:00
|
|
|
rewrite_bundles.len(),
|
|
|
|
to_file_size(rewrite_data as u64)
|
2017-07-21 09:21:59 +00:00
|
|
|
);
|
2017-03-20 17:11:03 +00:00
|
|
|
if !force {
|
2017-04-10 18:15:13 +00:00
|
|
|
self.dirty = false;
|
2017-07-21 09:21:59 +00:00
|
|
|
return Ok(());
|
2017-03-20 13:03:29 +00:00
|
|
|
}
|
2017-07-21 09:21:59 +00:00
|
|
|
for id in ProgressIter::new(
|
|
|
|
"rewriting bundles",
|
|
|
|
rewrite_bundles.len(),
|
|
|
|
rewrite_bundles.iter()
|
|
|
|
)
|
|
|
|
{
|
2017-03-20 17:11:03 +00:00
|
|
|
let bundle = &usage[id];
|
2017-03-22 11:27:17 +00:00
|
|
|
let bundle_id = self.bundle_map.get(*id).unwrap();
|
2017-03-24 11:52:01 +00:00
|
|
|
let chunks = try!(self.bundles.get_chunk_list(&bundle_id));
|
2017-03-25 11:43:49 +00:00
|
|
|
let mode = usage[id].info.mode;
|
2017-03-24 11:52:01 +00:00
|
|
|
for (chunk, &(hash, _len)) in chunks.into_iter().enumerate() {
|
2017-03-25 11:43:49 +00:00
|
|
|
if !bundle.chunk_usage.get(chunk) {
|
2017-03-20 13:03:29 +00:00
|
|
|
try!(self.index.delete(&hash));
|
2017-07-21 09:21:59 +00:00
|
|
|
continue;
|
2017-03-20 13:03:29 +00:00
|
|
|
}
|
2017-03-24 11:52:01 +00:00
|
|
|
let data = try!(self.bundles.get_chunk(&bundle_id, chunk));
|
2017-03-20 13:03:29 +00:00
|
|
|
try!(self.put_chunk_override(mode, hash, &data));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
try!(self.flush());
|
|
|
|
info!("Checking index");
|
2017-04-18 11:52:43 +00:00
|
|
|
for (hash, location) in self.index.iter() {
|
2018-02-19 21:30:59 +00:00
|
|
|
let loc_bundle = location.bundle;
|
|
|
|
let loc_chunk = location.chunk;
|
|
|
|
if rewrite_bundles.contains(&loc_bundle) {
|
2017-07-21 09:21:59 +00:00
|
|
|
panic!(
|
|
|
|
"Removed bundle is still referenced in index: hash:{}, bundle:{}, chunk:{}",
|
|
|
|
hash,
|
2018-02-19 21:30:59 +00:00
|
|
|
loc_bundle,
|
|
|
|
loc_chunk
|
2017-07-21 09:21:59 +00:00
|
|
|
);
|
2017-03-20 13:03:29 +00:00
|
|
|
}
|
2017-04-18 11:52:43 +00:00
|
|
|
}
|
2017-03-20 13:03:29 +00:00
|
|
|
info!("Deleting {} bundles", rewrite_bundles.len());
|
|
|
|
for id in rewrite_bundles {
|
|
|
|
try!(self.delete_bundle(id));
|
|
|
|
}
|
2017-04-08 12:35:10 +00:00
|
|
|
try!(self.save_bundle_map());
|
2017-04-10 18:15:13 +00:00
|
|
|
self.dirty = false;
|
2017-03-20 13:03:29 +00:00
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
}
|