sausage core: modules per topic, journal writer thread with WAL and batches, hash cache by metadata, 49 tests per rule
Blank stdout lines give no warning. core.exit requests the exit in its own handler.
This commit is contained in:
+142
-26
@@ -1,42 +1,158 @@
|
||||
//! SQLite journal entries and filters.
|
||||
//! A dedicated SQLite writer batches entries and serves ordered journal queries.
|
||||
use rusqlite::{Connection, params};
|
||||
use serde_json::{Value, json};
|
||||
use std::{collections::HashSet, path::Path, sync::Mutex};
|
||||
pub struct Journal {
|
||||
db: Mutex<Connection>,
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
path::PathBuf,
|
||||
sync::{Mutex, mpsc},
|
||||
};
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
struct Entry {
|
||||
id: String,
|
||||
pid: String,
|
||||
plugin: String,
|
||||
stream: String,
|
||||
line: String,
|
||||
}
|
||||
enum Command {
|
||||
Write(Entry),
|
||||
Query(Value, oneshot::Sender<Value>),
|
||||
}
|
||||
|
||||
pub(crate) struct Journal {
|
||||
sender: mpsc::Sender<Command>,
|
||||
warned: Mutex<HashSet<String>>,
|
||||
}
|
||||
impl Journal {
|
||||
pub fn open(path: &Path) -> anyhow::Result<Self> {
|
||||
std::fs::create_dir_all(path.parent().unwrap())?;
|
||||
let db = Connection::open(path)?;
|
||||
db.execute_batch("CREATE TABLE IF NOT EXISTS journal (id TEXT PRIMARY KEY, pid TEXT, plugin TEXT, stream TEXT, line TEXT)")?;
|
||||
/// Open SQLite on its owning thread and wait for its schema to become ready.
|
||||
pub(crate) async fn open(path: PathBuf) -> anyhow::Result<Self> {
|
||||
let (sender, receiver) = mpsc::channel();
|
||||
let (ready, initialized) = oneshot::channel();
|
||||
std::thread::spawn(move || {
|
||||
let opened = open_database(path);
|
||||
match opened {
|
||||
Ok(mut db) => {
|
||||
let _ = ready.send(Ok(()));
|
||||
while let Ok(first) = receiver.recv() {
|
||||
let mut batch = vec![first];
|
||||
batch.extend(receiver.try_iter().take(255));
|
||||
write_batch(&mut db, batch);
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = ready.send(Err(error));
|
||||
}
|
||||
}
|
||||
});
|
||||
initialized.await??;
|
||||
Ok(Self {
|
||||
db: Mutex::new(db),
|
||||
sender,
|
||||
warned: Mutex::new(HashSet::new()),
|
||||
})
|
||||
}
|
||||
pub fn write(&self, pid: &str, plugin: &str, stream: &str, line: &str) {
|
||||
self.db
|
||||
.lock()
|
||||
.unwrap()
|
||||
.execute(
|
||||
"INSERT INTO journal VALUES (?1,?2,?3,?4,?5)",
|
||||
params![uuid::Uuid::now_v7().to_string(), pid, plugin, stream, line],
|
||||
)
|
||||
.expect("write journal");
|
||||
/// Queue an entry without file operations or SQLite work on the caller's thread.
|
||||
pub(crate) fn write(&self, pid: &str, plugin: &str, stream: &str, line: &str) {
|
||||
let entry = Entry {
|
||||
id: uuid::Uuid::now_v7().to_string(),
|
||||
pid: pid.into(),
|
||||
plugin: plugin.into(),
|
||||
stream: stream.into(),
|
||||
line: line.into(),
|
||||
};
|
||||
self.sender
|
||||
.send(Command::Write(entry))
|
||||
.expect("journal writer stopped");
|
||||
}
|
||||
pub fn warning(&self, pid: &str, plugin: &str) {
|
||||
/// Queue one warning for the first non-blank discarded line of a process.
|
||||
pub(crate) fn warning(&self, pid: &str, plugin: &str) {
|
||||
if self.warned.lock().unwrap().insert(pid.into()) {
|
||||
self.write(pid, plugin, "core", "warning: discarded stdout");
|
||||
}
|
||||
}
|
||||
pub fn query(&self, p: &Value) -> Value {
|
||||
let db = self.db.lock().unwrap();
|
||||
let mut stmt=db.prepare("SELECT id,pid,plugin,stream,line FROM journal WHERE (?1 IS NULL OR plugin=?1) AND (?2 IS NULL OR pid=?2) AND (?3 IS NULL OR id>=?3) AND (?4 IS NULL OR id<=?4) AND (?5 IS NULL OR instr(line,?5)>0) ORDER BY id DESC LIMIT ?6").unwrap();
|
||||
let rows=stmt.query_map(params![p["plugin"].as_str(),p["pid"].as_str(),p["since"].as_str(),p["until"].as_str(),p["grep"].as_str(),p["limit"].as_i64().unwrap_or(100)],|r|Ok(json!({"id":r.get::<_,String>(0)?,"pid":r.get::<_,String>(1)?,"plugin":r.get::<_,String>(2)?,"stream":r.get::<_,String>(3)?,"line":r.get::<_,String>(4)?}))).unwrap();
|
||||
let mut entries: Vec<_> = rows.map(Result::unwrap).collect();
|
||||
entries.reverse();
|
||||
json!(entries)
|
||||
/// Queue a query after all entries that the caller already submitted.
|
||||
pub(crate) fn query_later(&self, filters: Value) -> oneshot::Receiver<Value> {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
self.sender
|
||||
.send(Command::Query(filters, tx))
|
||||
.expect("journal writer stopped");
|
||||
rx
|
||||
}
|
||||
/// Wait asynchronously for the journal thread to execute a query.
|
||||
pub(crate) async fn query(&self, filters: Value) -> Value {
|
||||
self.query_later(filters)
|
||||
.await
|
||||
.expect("journal writer stopped")
|
||||
}
|
||||
}
|
||||
fn open_database(path: PathBuf) -> anyhow::Result<Connection> {
|
||||
std::fs::create_dir_all(path.parent().unwrap())?;
|
||||
let db = Connection::open(path)?;
|
||||
db.execute_batch(
|
||||
"PRAGMA journal_mode=WAL;
|
||||
PRAGMA synchronous=NORMAL;
|
||||
CREATE TABLE IF NOT EXISTS journal (
|
||||
id TEXT PRIMARY KEY, time TEXT, pid TEXT, plugin TEXT, stream TEXT, line TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS journal_pid ON journal(pid);
|
||||
CREATE INDEX IF NOT EXISTS journal_plugin ON journal(plugin);",
|
||||
)?;
|
||||
Ok(db)
|
||||
}
|
||||
/// Commit queued writes together before answering the batch's queries.
|
||||
fn write_batch(db: &mut Connection, batch: Vec<Command>) {
|
||||
let transaction = db.transaction().expect("journal transaction");
|
||||
for command in &batch {
|
||||
if let Command::Write(entry) = command {
|
||||
transaction
|
||||
.execute(
|
||||
"INSERT INTO journal (id,time,pid,plugin,stream,line)
|
||||
VALUES (?1,strftime('%Y-%m-%dT%H:%M:%fZ','now'),?2,?3,?4,?5)",
|
||||
params![entry.id, entry.pid, entry.plugin, entry.stream, entry.line],
|
||||
)
|
||||
.expect("write journal");
|
||||
}
|
||||
}
|
||||
transaction.commit().expect("commit journal");
|
||||
for command in batch {
|
||||
if let Command::Query(filters, response) = command {
|
||||
let _ = response.send(query_database(db, &filters));
|
||||
}
|
||||
}
|
||||
}
|
||||
fn query_database(db: &Connection, filters: &Value) -> Value {
|
||||
let mut statement = db
|
||||
.prepare(
|
||||
"SELECT id,time,pid,plugin,stream,line FROM journal
|
||||
WHERE (?1 IS NULL OR plugin=?1) AND (?2 IS NULL OR pid=?2)
|
||||
AND (?3 IS NULL OR id>=?3) AND (?4 IS NULL OR id<=?4)
|
||||
AND (?5 IS NULL OR instr(line,?5)>0)
|
||||
ORDER BY id DESC LIMIT ?6",
|
||||
)
|
||||
.expect("prepare journal query");
|
||||
let rows = statement
|
||||
.query_map(
|
||||
params![
|
||||
filters["plugin"].as_str(),
|
||||
filters["pid"].as_str(),
|
||||
filters["since"].as_str(),
|
||||
filters["until"].as_str(),
|
||||
filters["grep"].as_str(),
|
||||
filters["limit"].as_i64().unwrap_or(100)
|
||||
],
|
||||
|row| {
|
||||
Ok(json!({
|
||||
"id":row.get::<_,String>(0)?,
|
||||
"time":row.get::<_,String>(1)?,
|
||||
"pid":row.get::<_,String>(2)?,
|
||||
"plugin":row.get::<_,String>(3)?,
|
||||
"stream":row.get::<_,String>(4)?,
|
||||
"line":row.get::<_,String>(5)?,
|
||||
}))
|
||||
},
|
||||
)
|
||||
.expect("query journal");
|
||||
let mut entries: Vec<_> = rows.map(Result::unwrap).collect();
|
||||
entries.reverse();
|
||||
json!(entries)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user