lmdb/
environment.rs

1use libc::{
2    c_uint,
3    size_t,
4};
5use std::ffi::CString;
6#[cfg(windows)]
7use std::ffi::OsStr;
8#[cfg(unix)]
9use std::os::unix::ffi::OsStrExt;
10use std::path::Path;
11use std::sync::Mutex;
12use std::{
13    fmt,
14    mem,
15    ptr,
16    result,
17};
18
19use ffi;
20
21use byteorder::{
22    ByteOrder,
23    NativeEndian,
24};
25
26use cursor::Cursor;
27use database::Database;
28use error::{
29    lmdb_result,
30    Error,
31    Result,
32};
33use flags::{
34    DatabaseFlags,
35    EnvironmentFlags,
36};
37use transaction::{
38    RoTransaction,
39    RwTransaction,
40    Transaction,
41};
42
43#[cfg(windows)]
44/// Adding a 'missing' trait from windows OsStrExt
45trait OsStrExtLmdb {
46    fn as_bytes(&self) -> &[u8];
47}
48#[cfg(windows)]
49impl OsStrExtLmdb for OsStr {
50    fn as_bytes(&self) -> &[u8] {
51        &self.to_str().unwrap().as_bytes()
52    }
53}
54
55/// An LMDB environment.
56///
57/// An environment supports multiple databases, all residing in the same shared-memory map.
58pub struct Environment {
59    env: *mut ffi::MDB_env,
60    dbi_open_mutex: Mutex<()>,
61}
62
63impl Environment {
64    /// Creates a new builder for specifying options for opening an LMDB environment.
65    #[allow(clippy::new_ret_no_self)]
66    pub fn new() -> EnvironmentBuilder {
67        EnvironmentBuilder {
68            flags: EnvironmentFlags::empty(),
69            max_readers: None,
70            max_dbs: None,
71            map_size: None,
72        }
73    }
74
75    /// Returns a raw pointer to the underlying LMDB environment.
76    ///
77    /// The caller **must** ensure that the pointer is not dereferenced after the lifetime of the
78    /// environment.
79    pub fn env(&self) -> *mut ffi::MDB_env {
80        self.env
81    }
82
83    /// Opens a handle to an LMDB database.
84    ///
85    /// If `name` is `None`, then the returned handle will be for the default database.
86    ///
87    /// If `name` is not `None`, then the returned handle will be for a named database. In this
88    /// case the environment must be configured to allow named databases through
89    /// `EnvironmentBuilder::set_max_dbs`.
90    ///
91    /// The returned database handle may be shared among any transaction in the environment.
92    ///
93    /// This function will fail with `Error::BadRslot` if called by a thread which has an ongoing
94    /// transaction.
95    ///
96    /// The database name may not contain the null character.
97    pub fn open_db<'env>(&'env self, name: Option<&str>) -> Result<Database> {
98        let mutex = self.dbi_open_mutex.lock();
99        let txn = self.begin_ro_txn()?;
100        let db = unsafe { txn.open_db(name)? };
101        txn.commit()?;
102        drop(mutex);
103        Ok(db)
104    }
105
106    /// Opens a handle to an LMDB database, creating the database if necessary.
107    ///
108    /// If the database is already created, the given option flags will be added to it.
109    ///
110    /// If `name` is `None`, then the returned handle will be for the default database.
111    ///
112    /// If `name` is not `None`, then the returned handle will be for a named database. In this
113    /// case the environment must be configured to allow named databases through
114    /// `EnvironmentBuilder::set_max_dbs`.
115    ///
116    /// The returned database handle may be shared among any transaction in the environment.
117    ///
118    /// This function will fail with `Error::BadRslot` if called by a thread with an open
119    /// transaction.
120    pub fn create_db<'env>(&'env self, name: Option<&str>, flags: DatabaseFlags) -> Result<Database> {
121        let mutex = self.dbi_open_mutex.lock();
122        let txn = self.begin_rw_txn()?;
123        let db = unsafe { txn.create_db(name, flags)? };
124        txn.commit()?;
125        drop(mutex);
126        Ok(db)
127    }
128
129    /// Retrieves the set of flags which the database is opened with.
130    ///
131    /// The database must belong to to this environment.
132    pub fn get_db_flags(&self, db: Database) -> Result<DatabaseFlags> {
133        let txn = self.begin_ro_txn()?;
134        let mut flags: c_uint = 0;
135        unsafe {
136            lmdb_result(ffi::mdb_dbi_flags(txn.txn(), db.dbi(), &mut flags))?;
137        }
138        Ok(DatabaseFlags::from_bits(flags).unwrap())
139    }
140
141    /// Create a read-only transaction for use with the environment.
142    pub fn begin_ro_txn<'env>(&'env self) -> Result<RoTransaction<'env>> {
143        RoTransaction::new(self)
144    }
145
146    /// Create a read-write transaction for use with the environment. This method will block while
147    /// there are any other read-write transactions open on the environment.
148    pub fn begin_rw_txn<'env>(&'env self) -> Result<RwTransaction<'env>> {
149        RwTransaction::new(self)
150    }
151
152    /// Flush data buffers to disk.
153    ///
154    /// Data is always written to disk when `Transaction::commit` is called, but the operating
155    /// system may keep it buffered. LMDB always flushes the OS buffers upon commit as well, unless
156    /// the environment was opened with `MDB_NOSYNC` or in part `MDB_NOMETASYNC`.
157    pub fn sync(&self, force: bool) -> Result<()> {
158        unsafe {
159            lmdb_result(ffi::mdb_env_sync(
160                self.env(),
161                if force {
162                    1
163                } else {
164                    0
165                },
166            ))
167        }
168    }
169
170    /// Closes the database handle. Normally unnecessary.
171    ///
172    /// Closing a database handle is not necessary, but lets `Transaction::open_database` reuse the
173    /// handle value. Usually it's better to set a bigger `EnvironmentBuilder::set_max_dbs`, unless
174    /// that value would be large.
175    ///
176    /// ## Safety
177    ///
178    /// This call is not mutex protected. Databases should only be closed by a single thread, and
179    /// only if no other threads are going to reference the database handle or one of its cursors
180    /// any further. Do not close a handle if an existing transaction has modified its database.
181    /// Doing so can cause misbehavior from database corruption to errors like
182    /// `Error::BadValSize` (since the DB name is gone).
183    pub unsafe fn close_db(&mut self, db: Database) {
184        ffi::mdb_dbi_close(self.env, db.dbi());
185    }
186
187    /// Retrieves statistics about this environment.
188    pub fn stat(&self) -> Result<Stat> {
189        unsafe {
190            let mut stat = Stat::new();
191            lmdb_try!(ffi::mdb_env_stat(self.env(), stat.mdb_stat()));
192            Ok(stat)
193        }
194    }
195
196    /// Retrieves info about this environment.
197    pub fn info(&self) -> Result<Info> {
198        unsafe {
199            let mut info = Info(mem::zeroed());
200            lmdb_try!(ffi::mdb_env_info(self.env(), &mut info.0));
201            Ok(info)
202        }
203    }
204
205    /// Retrieves the total number of pages on the freelist.
206    ///
207    /// Along with `Environment::info()`, this can be used to calculate the exact number
208    /// of used pages as well as free pages in this environment.
209    ///
210    /// ```ignore
211    /// let env = Environment::new().open("/tmp/test").unwrap();
212    /// let info = env.info().unwrap();
213    /// let stat = env.stat().unwrap();
214    /// let freelist = env.freelist().unwrap();
215    /// let last_pgno = info.last_pgno() + 1; // pgno is 0 based.
216    /// let total_pgs = info.map_size() / stat.page_size() as usize;
217    /// let pgs_in_use = last_pgno - freelist;
218    /// let pgs_free = total_pgs - pgs_in_use;
219    /// ```
220    ///
221    /// Note:
222    ///
223    /// * LMDB stores all the freelists in the designated database 0 in each environment,
224    ///   and the freelist count is stored at the beginning of the value as `libc::size_t`
225    ///   in the native byte order.
226    ///
227    /// * It will create a read transaction to traverse the freelist database.
228    pub fn freelist(&self) -> Result<size_t> {
229        let mut freelist: size_t = 0;
230        let db = Database::freelist_db();
231        let txn = self.begin_ro_txn()?;
232        let mut cursor = txn.open_ro_cursor(db)?;
233
234        for result in cursor.iter() {
235            let (_key, value) = result?;
236            if value.len() < mem::size_of::<size_t>() {
237                return Err(Error::Corrupted);
238            }
239
240            let s = &value[..mem::size_of::<size_t>()];
241            if cfg!(target_pointer_width = "64") {
242                freelist += NativeEndian::read_u64(s) as size_t;
243            } else {
244                freelist += NativeEndian::read_u32(s) as size_t;
245            }
246        }
247
248        Ok(freelist)
249    }
250
251    /// Sets the size of the memory map to use for the environment.
252    ///
253    /// This could be used to resize the map when the environment is already open.
254    ///
255    /// Note:
256    ///
257    /// * No active transactions allowed when performing resizing in this process.
258    ///
259    /// * The size should be a multiple of the OS page size. Any attempt to set
260    ///   a size smaller than the space already consumed by the environment will
261    ///   be silently changed to the current size of the used space.
262    ///
263    /// * In the multi-process case, once a process resizes the map, other
264    ///   processes need to either re-open the environment, or call set_map_size
265    ///   with size 0 to update the environment. Otherwise, new transaction creation
266    ///   will fail with `Error::MapResized`.
267    pub fn set_map_size(&self, size: size_t) -> Result<()> {
268        unsafe { lmdb_result(ffi::mdb_env_set_mapsize(self.env(), size)) }
269    }
270}
271
272/// Environment statistics.
273///
274/// Contains information about the size and layout of an LMDB environment or database.
275pub struct Stat(ffi::MDB_stat);
276
277impl Stat {
278    /// Create a new Stat with zero'd inner struct `ffi::MDB_stat`.
279    pub(crate) fn new() -> Stat {
280        unsafe { Stat(mem::zeroed()) }
281    }
282
283    /// Returns a mut pointer to `ffi::MDB_stat`.
284    pub(crate) fn mdb_stat(&mut self) -> *mut ffi::MDB_stat {
285        &mut self.0
286    }
287}
288
289impl Stat {
290    /// Size of a database page. This is the same for all databases in the environment.
291    #[inline]
292    pub fn page_size(&self) -> u32 {
293        self.0.ms_psize
294    }
295
296    /// Depth (height) of the B-tree.
297    #[inline]
298    pub fn depth(&self) -> u32 {
299        self.0.ms_depth
300    }
301
302    /// Number of internal (non-leaf) pages.
303    #[inline]
304    pub fn branch_pages(&self) -> usize {
305        self.0.ms_branch_pages
306    }
307
308    /// Number of leaf pages.
309    #[inline]
310    pub fn leaf_pages(&self) -> usize {
311        self.0.ms_leaf_pages
312    }
313
314    /// Number of overflow pages.
315    #[inline]
316    pub fn overflow_pages(&self) -> usize {
317        self.0.ms_overflow_pages
318    }
319
320    /// Number of data items.
321    #[inline]
322    pub fn entries(&self) -> usize {
323        self.0.ms_entries
324    }
325}
326
327/// Environment information.
328///
329/// Contains environment information about the map size, readers, last txn id etc.
330pub struct Info(ffi::MDB_envinfo);
331
332impl Info {
333    /// Size of memory map.
334    #[inline]
335    pub fn map_size(&self) -> usize {
336        self.0.me_mapsize
337    }
338
339    /// Last used page number
340    #[inline]
341    pub fn last_pgno(&self) -> usize {
342        self.0.me_last_pgno
343    }
344
345    /// Last transaction ID
346    #[inline]
347    pub fn last_txnid(&self) -> usize {
348        self.0.me_last_txnid
349    }
350
351    /// Max reader slots in the environment
352    #[inline]
353    pub fn max_readers(&self) -> u32 {
354        self.0.me_maxreaders
355    }
356
357    /// Max reader slots used in the environment
358    #[inline]
359    pub fn num_readers(&self) -> u32 {
360        self.0.me_numreaders
361    }
362}
363
364unsafe impl Send for Environment {}
365unsafe impl Sync for Environment {}
366
367impl fmt::Debug for Environment {
368    fn fmt(&self, f: &mut fmt::Formatter) -> result::Result<(), fmt::Error> {
369        f.debug_struct("Environment").finish()
370    }
371}
372
373impl Drop for Environment {
374    fn drop(&mut self) {
375        unsafe { ffi::mdb_env_close(self.env) }
376    }
377}
378
379///////////////////////////////////////////////////////////////////////////////////////////////////
380//// Environment Builder
381///////////////////////////////////////////////////////////////////////////////////////////////////
382
383/// Options for opening or creating an environment.
384#[derive(Debug, PartialEq, Eq, Copy, Clone)]
385pub struct EnvironmentBuilder {
386    flags: EnvironmentFlags,
387    max_readers: Option<c_uint>,
388    max_dbs: Option<c_uint>,
389    map_size: Option<size_t>,
390}
391
392impl EnvironmentBuilder {
393    /// Open an environment.
394    ///
395    /// On UNIX, the database files will be opened with 644 permissions.
396    ///
397    /// The path may not contain the null character, Windows UNC (Uniform Naming Convention)
398    /// paths are not supported either.
399    pub fn open(&self, path: &Path) -> Result<Environment> {
400        self.open_with_permissions(path, 0o644)
401    }
402
403    /// Open an environment with the provided UNIX permissions.
404    ///
405    /// On Windows, the permissions will be ignored.
406    ///
407    /// The path may not contain the null character, Windows UNC (Uniform Naming Convention)
408    /// paths are not supported either.
409    pub fn open_with_permissions(&self, path: &Path, mode: ffi::mdb_mode_t) -> Result<Environment> {
410        let mut env: *mut ffi::MDB_env = ptr::null_mut();
411        unsafe {
412            lmdb_try!(ffi::mdb_env_create(&mut env));
413            if let Some(max_readers) = self.max_readers {
414                lmdb_try_with_cleanup!(ffi::mdb_env_set_maxreaders(env, max_readers), ffi::mdb_env_close(env))
415            }
416            if let Some(max_dbs) = self.max_dbs {
417                lmdb_try_with_cleanup!(ffi::mdb_env_set_maxdbs(env, max_dbs), ffi::mdb_env_close(env))
418            }
419            if let Some(map_size) = self.map_size {
420                lmdb_try_with_cleanup!(ffi::mdb_env_set_mapsize(env, map_size), ffi::mdb_env_close(env))
421            }
422            let path = match CString::new(path.as_os_str().as_bytes()) {
423                Ok(path) => path,
424                Err(..) => return Err(::Error::Invalid),
425            };
426            lmdb_try_with_cleanup!(
427                ffi::mdb_env_open(env, path.as_ptr(), self.flags.bits(), mode),
428                ffi::mdb_env_close(env)
429            );
430        }
431        Ok(Environment {
432            env,
433            dbi_open_mutex: Mutex::new(()),
434        })
435    }
436
437    /// Sets the provided options in the environment.
438    pub fn set_flags(&mut self, flags: EnvironmentFlags) -> &mut EnvironmentBuilder {
439        self.flags = flags;
440        self
441    }
442
443    /// Sets the maximum number of threads or reader slots for the environment.
444    ///
445    /// This defines the number of slots in the lock table that is used to track readers in the
446    /// the environment. The default is 126. Starting a read-only transaction normally ties a lock
447    /// table slot to the current thread until the environment closes or the thread exits. If
448    /// `MDB_NOTLS` is in use, `Environment::open_txn` instead ties the slot to the `Transaction`
449    /// object until it or the `Environment` object is destroyed.
450    pub fn set_max_readers(&mut self, max_readers: c_uint) -> &mut EnvironmentBuilder {
451        self.max_readers = Some(max_readers);
452        self
453    }
454
455    /// Sets the maximum number of named databases for the environment.
456    ///
457    /// This function is only needed if multiple databases will be used in the
458    /// environment. Simpler applications that use the environment as a single
459    /// unnamed database can ignore this option.
460    ///
461    /// Currently a moderate number of slots are cheap but a huge number gets
462    /// expensive: 7-120 words per transaction, and every `Transaction::open_db`
463    /// does a linear search of the opened slots.
464    pub fn set_max_dbs(&mut self, max_dbs: c_uint) -> &mut EnvironmentBuilder {
465        self.max_dbs = Some(max_dbs);
466        self
467    }
468
469    /// Sets the size of the memory map to use for the environment.
470    ///
471    /// The size should be a multiple of the OS page size. The default is
472    /// 1048576 bytes. The size of the memory map is also the maximum size
473    /// of the database. The value should be chosen as large as possible,
474    /// to accommodate future growth of the database. It may be increased at
475    /// later times.
476    ///
477    /// Any attempt to set a size smaller than the space already consumed
478    /// by the environment will be silently changed to the current size of the used space.
479    pub fn set_map_size(&mut self, map_size: size_t) -> &mut EnvironmentBuilder {
480        self.map_size = Some(map_size);
481        self
482    }
483}
484
485#[cfg(test)]
486mod test {
487
488    extern crate byteorder;
489
490    use self::byteorder::{
491        ByteOrder,
492        LittleEndian,
493    };
494    use tempdir::TempDir;
495
496    use flags::*;
497
498    use super::*;
499
500    #[test]
501    fn test_open() {
502        let dir = TempDir::new("test").unwrap();
503
504        // opening non-existent env with read-only should fail
505        assert!(Environment::new().set_flags(EnvironmentFlags::READ_ONLY).open(dir.path()).is_err());
506
507        // opening non-existent env should succeed
508        assert!(Environment::new().open(dir.path()).is_ok());
509
510        // opening env with read-only should succeed
511        assert!(Environment::new().set_flags(EnvironmentFlags::READ_ONLY).open(dir.path()).is_ok());
512    }
513
514    #[test]
515    fn test_begin_txn() {
516        let dir = TempDir::new("test").unwrap();
517
518        {
519            // writable environment
520            let env = Environment::new().open(dir.path()).unwrap();
521
522            assert!(env.begin_rw_txn().is_ok());
523            assert!(env.begin_ro_txn().is_ok());
524        }
525
526        {
527            // read-only environment
528            let env = Environment::new().set_flags(EnvironmentFlags::READ_ONLY).open(dir.path()).unwrap();
529
530            assert!(env.begin_rw_txn().is_err());
531            assert!(env.begin_ro_txn().is_ok());
532        }
533    }
534
535    #[test]
536    fn test_open_db() {
537        let dir = TempDir::new("test").unwrap();
538        let env = Environment::new().set_max_dbs(1).open(dir.path()).unwrap();
539
540        assert!(env.open_db(None).is_ok());
541        assert!(env.open_db(Some("testdb")).is_err());
542    }
543
544    #[test]
545    fn test_create_db() {
546        let dir = TempDir::new("test").unwrap();
547        let env = Environment::new().set_max_dbs(11).open(dir.path()).unwrap();
548        assert!(env.open_db(Some("testdb")).is_err());
549        assert!(env.create_db(Some("testdb"), DatabaseFlags::empty()).is_ok());
550        assert!(env.open_db(Some("testdb")).is_ok())
551    }
552
553    #[test]
554    fn test_close_database() {
555        let dir = TempDir::new("test").unwrap();
556        let mut env = Environment::new().set_max_dbs(10).open(dir.path()).unwrap();
557
558        let db = env.create_db(Some("db"), DatabaseFlags::empty()).unwrap();
559        unsafe {
560            env.close_db(db);
561        }
562        assert!(env.open_db(Some("db")).is_ok());
563    }
564
565    #[test]
566    fn test_sync() {
567        let dir = TempDir::new("test").unwrap();
568        {
569            let env = Environment::new().open(dir.path()).unwrap();
570            assert!(env.sync(true).is_ok());
571        }
572        {
573            let env = Environment::new().set_flags(EnvironmentFlags::READ_ONLY).open(dir.path()).unwrap();
574            assert!(env.sync(true).is_err());
575        }
576    }
577
578    #[test]
579    fn test_stat() {
580        let dir = TempDir::new("test").unwrap();
581        let env = Environment::new().open(dir.path()).unwrap();
582
583        // Stats should be empty initially.
584        let stat = env.stat().unwrap();
585        assert_eq!(stat.page_size(), 4096);
586        assert_eq!(stat.depth(), 0);
587        assert_eq!(stat.branch_pages(), 0);
588        assert_eq!(stat.leaf_pages(), 0);
589        assert_eq!(stat.overflow_pages(), 0);
590        assert_eq!(stat.entries(), 0);
591
592        let db = env.open_db(None).unwrap();
593
594        // Write a few small values.
595        for i in 0..64 {
596            let mut value = [0u8; 8];
597            LittleEndian::write_u64(&mut value, i);
598            let mut tx = env.begin_rw_txn().expect("begin_rw_txn");
599            tx.put(db, &value, &value, WriteFlags::default()).expect("tx.put");
600            tx.commit().expect("tx.commit")
601        }
602
603        // Stats should now reflect inserted values.
604        let stat = env.stat().unwrap();
605        assert_eq!(stat.page_size(), 4096);
606        assert_eq!(stat.depth(), 1);
607        assert_eq!(stat.branch_pages(), 0);
608        assert_eq!(stat.leaf_pages(), 1);
609        assert_eq!(stat.overflow_pages(), 0);
610        assert_eq!(stat.entries(), 64);
611    }
612
613    #[test]
614    fn test_info() {
615        let map_size = 1024 * 1024;
616        let dir = TempDir::new("test").unwrap();
617        let env = Environment::new().set_map_size(map_size).open(dir.path()).unwrap();
618
619        let info = env.info().unwrap();
620        assert_eq!(info.map_size(), map_size);
621        assert_eq!(info.last_pgno(), 1);
622        assert_eq!(info.last_txnid(), 0);
623        // The default max readers is 126.
624        assert_eq!(info.max_readers(), 126);
625        assert_eq!(info.num_readers(), 0);
626    }
627
628    #[test]
629    fn test_freelist() {
630        let dir = TempDir::new("test").unwrap();
631        let env = Environment::new().open(dir.path()).unwrap();
632
633        let db = env.open_db(None).unwrap();
634        let mut freelist = env.freelist().unwrap();
635        assert_eq!(freelist, 0);
636
637        // Write a few small values.
638        for i in 0..64 {
639            let mut value = [0u8; 8];
640            LittleEndian::write_u64(&mut value, i);
641            let mut tx = env.begin_rw_txn().expect("begin_rw_txn");
642            tx.put(db, &value, &value, WriteFlags::default()).expect("tx.put");
643            tx.commit().expect("tx.commit")
644        }
645        let mut tx = env.begin_rw_txn().expect("begin_rw_txn");
646        tx.clear_db(db).expect("clear");
647        tx.commit().expect("tx.commit");
648
649        // Freelist should not be empty after clear_db.
650        freelist = env.freelist().unwrap();
651        assert!(freelist > 0);
652    }
653
654    #[test]
655    fn test_set_map_size() {
656        let dir = TempDir::new("test").unwrap();
657        let env = Environment::new().open(dir.path()).unwrap();
658
659        let mut info = env.info().unwrap();
660        let default_size = info.map_size();
661
662        // Resizing to 0 merely reloads the map size
663        env.set_map_size(0).unwrap();
664        info = env.info().unwrap();
665        assert_eq!(info.map_size(), default_size);
666
667        env.set_map_size(2 * default_size).unwrap();
668        info = env.info().unwrap();
669        assert_eq!(info.map_size(), 2 * default_size);
670
671        env.set_map_size(4 * default_size).unwrap();
672        info = env.info().unwrap();
673        assert_eq!(info.map_size(), 4 * default_size);
674
675        // Decreasing is also fine if the space hasn't been consumed.
676        env.set_map_size(2 * default_size).unwrap();
677        info = env.info().unwrap();
678        assert_eq!(info.map_size(), 2 * default_size);
679    }
680}
lmdb/environment.rs

lmdb/
environment.rs
