Compare commits

..

68 Commits

Author SHA1 Message Date
Michael Holloway
b1db7f1a68 Changes to be committed:
modified:   mdlspipe
	Adding defaults to script to support bot .env and nomadic usage.
2022-03-29 22:14:29 -04:00
Michael Holloway
86a47efc6e On branch output-file
Changes to be committed:
renamed:    hash-files -> mdlspipe
2022-03-29 21:48:32 -04:00
Michael Holloway
9ffd223048 On branch docs-minimal
Changes to be committed:
new file:   LICENSE
	bsd-2clause
new file:   README.md
	Basics like summary and usage guidance.
2022-03-29 21:41:27 -04:00
Mike Holloway
b069d30fa9 On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Cleaning up comment lines
2022-03-26 19:44:40 -04:00
Mike Holloway
a889f1ff6f On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Change from two step (invoke, print) to one step (print
	invocation)
2022-03-26 19:42:50 -04:00
Mike Holloway
fb29201bcd Line 42 - Is this where the redundant mdls properties output comes from? 2022-03-26 19:36:31 -04:00
Mike Holloway
01faecccda On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Comment cleanup, output cleanup
2022-03-26 19:33:41 -04:00
Mike Holloway
ddc61edccd On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Finally, commas in-between the mdls values!
	Prepare the csv...
2022-03-26 19:28:20 -04:00
Mike Holloway
33e88fba83 modified: hash-files
Passing mdls_value to mdls_properties as a list instead of
		string variable
2022-03-26 19:26:52 -04:00
Mike Holloway
0ece9167c0 modified: hash-files 2022-03-26 19:25:59 -04:00
Mike Holloway
55be94f26a modified: hash-files 2022-03-26 19:23:47 -04:00
Mike Holloway
ac974fff80 Still troubleshooting output 2022-03-26 19:22:36 -04:00
Mike Holloway
bbc65490d3 Can we add the IFS work now? 2022-03-26 19:19:24 -04:00
Mike Holloway
f52295c6ee Adding echo in main to print rather than call.. whoops 2022-03-26 19:17:56 -04:00
Mike Holloway
320d899e95 Printing via function to debug main. 2022-03-26 19:16:36 -04:00
Mike Holloway
008de3f44c Export post-invocation of mdls_per_file function should be enough 2022-03-26 19:14:33 -04:00
Mike Holloway
652e775164 Changing where array invocation is done, still sorting out array output. 2022-03-26 19:13:14 -04:00
Mike Holloway
6a57064a74 echo vs printf for proper delimiter? Let's find out 2022-03-26 19:09:52 -04:00
Mike Holloway
fb583285e5 Even more IFS-array finagling 2022-03-26 19:07:41 -04:00
Mike Holloway
f71987d044 More IFS-in-array finagling 2022-03-26 19:06:45 -04:00
Mike Holloway
528bb7def0 IFS work for comma delimited arrays 2022-03-26 19:05:24 -04:00
Mike Holloway
16a6db49d9 whoops wrong bracket type. 2022-03-26 19:03:12 -04:00
Mike Holloway
2f5be070c8 On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Temp csv file (re)integration
2022-03-26 19:02:17 -04:00
Mike Holloway
e21989a5c3 On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	mdls_properties is accumulating values. Working on resetting
		per-file
2022-03-26 18:57:58 -04:00
Mike Holloway
60568fe7a8 Changing how mdls_properties is passed to output. 2022-03-26 18:56:41 -04:00
Mike Holloway
021d7c695a Deref'd when I should have ref'd. 2022-03-26 18:55:01 -04:00
Mike Holloway
316f57e41f On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Constructing mdls_properties array.
2022-03-26 18:54:09 -04:00
Mike Holloway
8d391800d6 Using the wrong array reference syntax? Let's find out 2022-03-26 18:49:33 -04:00
Mike Holloway
d602c76796 ....? 2022-03-26 18:48:08 -04:00
Mike Holloway
dbfe55872f Do I have it? 2022-03-26 18:47:02 -04:00
Mike Holloway
1fd34742fe Still getting duplicate keys. Investigating. 2022-03-26 18:45:56 -04:00
Mike Holloway
3ac7a29917 Teasing out duplicate key... 2022-03-26 18:43:44 -04:00
Mike Holloway
018d832456 Further clarification of mdls_properties list. 2022-03-26 18:41:50 -04:00
Mike Holloway
7d82f754db Debugging duplicate MDLS Column names 2022-03-26 18:40:31 -04:00
Mike Holloway
6f6fedcb95 Reintroducing filter 2022-03-26 18:37:40 -04:00
Mike Holloway
191d690603 Investigating main for loop and evocation of mdls_per_file 2022-03-26 18:34:55 -04:00
Mike Holloway
71e228c6c2 Iteration errors? Getting duplicate MDLS property column names? 2022-03-26 18:33:22 -04:00
Mike Holloway
2a491037a0 Improving debug output for mdls values as prepared for csv lines 2022-03-26 18:30:54 -04:00
Mike Holloway
05c46480fa On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Important to note - macOS `md5` the checksum is field 4
		whereas Linux `md5sum` the checksum is field 1
2022-03-26 18:28:28 -04:00
Mike Holloway
329f3a71ff On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Select the correct md5sum field with awk.
2022-03-26 18:27:17 -04:00
Mike Holloway
986aada28f Passing arguments to write_per_file 2022-03-26 18:26:02 -04:00
Mike Holloway
30d39a1060 Whoops, then I forgot I need dummy output for write_per_file! 2022-03-26 18:23:37 -04:00
Mike Holloway
bbc0007917 Whoops, empty function. 2022-03-26 18:22:02 -04:00
Mike Holloway
c9d13e3a25 On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Moving csv file preparation outside per-file loop.
2022-03-26 18:21:04 -04:00
Mike Holloway
f32e7b9242 On branch rewrite-2022-03-26
Changes to be committed:
modified:   hash-files
	Removing functions, refactoring, testing
2022-03-26 18:16:38 -04:00
Mike Holloway
271b922f1d On branch parse-mdls
Your branch is up to date with 'origin/parse-mdls'.

 Changes to be committed:
modified:   .env.default
	Adding default keys_of_interest value and cautions/explanations
2022-03-26 17:34:56 -04:00
Mike Holloway
1b6e51ef70 More do-nothing function fixes 2022-03-26 17:14:38 -04:00
Mike Holloway
7d671bccc0 Whoops, do-nothing function called. 2022-03-26 17:13:21 -04:00
Mike Holloway
e012a9b50f On branch parse-mdls
Your branch is up to date with 'origin/parse-mdls'.

 Changes to be committed:
modified:   hash-files
	Write csv columns to csv file.
2022-03-26 17:11:58 -04:00
Mike Holloway
fe2d738b42 On branch parse-mdls
Your branch is up to date with 'origin/parse-mdls'.

 Changes to be committed:
modified:   hash-files
	Added array delimiter replacement for csv-formatted output
2022-03-26 17:04:02 -04:00
Mike Holloway
5b81d72b2b On branch parse-mdls
Your branch is up to date with 'origin/parse-mdls'.

 Changes to be committed:
modified:   hash-files
	Drafting CSV format
2022-03-26 16:54:13 -04:00
Mike Holloway
d6bc8b84b3 On branch parse-mdls
Your branch is up to date with 'origin/parse-mdls'.

 Changes to be committed:
modified:   hash-files
	Added effective test for mdls_key in keys_of_interest.
2022-03-26 16:27:00 -04:00
Mike Holloway
2729af8349 Making carriage-return IFS global rather than subfunction specific 2022-03-26 16:17:47 -04:00
Mike Holloway
2da31a5355 Debugging mdls_files filter functionality 2022-03-26 15:22:42 -04:00
Mike Holloway
bf5ad573a2 Re-arranging grep in mdls_files if statement 2022-03-26 15:21:33 -04:00
Mike Holloway
e6e16d19cd Whoops, wrong brackets; changed '{}' to '()' 2022-03-26 15:20:10 -04:00
Mike Holloway
fc2dae9e56 On branch parse-mdls
Your branch is up to date with 'origin/parse-mdls'.

 Changes to be committed:
modified:   hash-files
	Added default keys_of_interest array
	Added output filter functionality leveraging keys_of_interest
2022-03-26 15:18:18 -04:00
Mike Holloway
8a9f48481f On branch parse-mdls
Your branch is up to date with 'origin/parse-mdls'.

 Changes to be committed:
modified:   hash-files
	Integrated functionality from mdls-parse subfunction into
	mdls_files
deleted:    mdls-parse
	Deleted; Integrated into hash-files
2022-03-26 15:09:22 -04:00
Mike Holloway
62f093dd89 Superflous k issue debugged - output is direct from mdls where 'k' is
introduced
2022-03-26 15:06:00 -04:00
Mike Holloway
6d20ec3a3c Debugging superflouous 'k' in output. 2022-03-26 15:01:43 -04:00
Mike Holloway
9c1dbe5860 Trying to eliminate superfluous 'k' before mdls_key variable output 2022-03-26 14:36:14 -04:00
Mike Holloway
9ad239a3dd Adding variables to printf command 2022-03-26 14:34:21 -04:00
Mike Holloway
2d210310b7 On branch parse-mdls
Your branch is up to date with 'origin/parse-mdls'.

 Changes to be committed:
modified:   mdls-parse
	Instantiating variables for mdls key,value pairs.
2022-03-26 14:33:05 -04:00
Mike Holloway
16ba557fd6 Whoops - cut not split 2022-03-26 14:25:17 -04:00
Mike Holloway
119db85b3a Add cut command using '=' as delimiter. 2022-03-26 14:24:09 -04:00
Mike Holloway
9c02331baa Setting IFS for proper loop output 2022-03-26 14:22:50 -04:00
Mike Holloway
4f391a62bd for loop echoing mdls output 2022-03-26 14:21:45 -04:00
Mike Holloway
9b0f67d81a On branch parse-mdls
Changes to be committed:
new file:   mdls-parse
	Bash subfunction to parse mdls data and produce data for SQL
	tables
2022-03-26 14:07:49 -04:00
5 changed files with 121 additions and 98 deletions

View File

@ -1,2 +1,7 @@
Working_Fullpath=$(pwd) Working_Fullpath=$(pwd)
Working_Directory=$(basename $Working_Fullpath) Working_Directory=$(basename $Working_Fullpath)
# IMPORTANT - The order of keys in keys_of_interest *must* match the order in
# which mdls prints them or csv output will be incoherent
keys_of_interest=("kMDItemFSCreationDate" "kMDItemFSName")

8
LICENSE Normal file
View File

@ -0,0 +1,8 @@
Copyright (c) Michael Holloway, 2022
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

21
README.md Normal file
View File

@ -0,0 +1,21 @@
mdlspipe
========
v0.1.1
Supported Platforms: Mac OS X
Provided a directory (default is current directory), capture particular
fields of `mdls` output for each file in provided directory in a csv.
Usage
-----
Run directly:
./mdlspipe
Place mdlspipe in your $PATH and run from $PWD.
By default, mdlspipe will write the output data to a csv called
'mdlspipe_YYYY-MM-DD_HH-MM-SS.csv'.

View File

@ -1,98 +0,0 @@
#!/usr/bin/env bash
log_file=/var/log/hash-files.log
md5_bin=$(which md5sum)
mktemp_bin=$(which mktemp)
DateTimeStamp=$(date +\%D_\%T)
set -a
source .env 2>&1 > /dev/null
set +a
## Functions
############
write_log () {
#echo " ++ Write to log.
# +++++++++++++++"
## Use on systems managed with systemd
# printf 'hash-files: %s\n' "$1" | systemd-cat
printf '%s hash-files: %s\n' "$DateTimeStamp" "$1" >> $log_file
}
create_list_of_files () {
#echo "=== Get list of files in $Working_Directory. ===
#=================================================="
IFS=$'\n'
for file in $(find $Working_Fullpath -maxdepth 1 -type f); do
list_of_files+=( $file )
done
write_log "List of files created successfully."
unset IFS
}
mdls_files () {
IFS=$'\n'
for file in $list_of_files; do
echo "=== mdls info ===
==========="
mdls $file
done
}
hash_files () {
#echo "=== Hash files. ===
#======================"
IFS=$'\n'
for file in ${list_of_files[@]}; do
hash=$($md5_bin $file | awk '{ print $1 }')
list_of_hashes+=( $hash )
done
write_log "Files hashed successfully."
unset IFS
}
create_csv () {
#echo "=== Create array of comma-separated-values. ===
#=================================================="
for i in ${!list_of_files[@]}; do
csv_array+=( ${list_of_files[i]},${list_of_hashes[i]} )
done
export csv_array
write_log "CSV Array creation successful."
}
write_data () {
#echo "=== Write data to a .csv file. ===
#===================================="
temp_csv_file=$($mktemp_bin)
printf '%s,%s\n' "File Path" "File Hash" >> $temp_csv_file
for line in ${csv_array[@]};do
printf '%s\n' $line
done >> $temp_csv_file
# cat $temp_csv_file
write_log "$temp_csv_file written successfully."
}
cleanup () {
write_log "Cleanup completed successfully."
}
## Main
#######
trap cleanup EXIT
echo "Main Start"
create_list_of_files
mdls_files
#hash_files
#if [ ${#list_of_files[@]} != ${#list_of_hashes[@]} ];then
# echo "Error! Number of files and hashes differs. Exiting..." && exit
#else
# create_csv
#fi
#write_data
#while $list_of_files >=1; do
#done

87
mdlspipe Executable file
View File

@ -0,0 +1,87 @@
#!/usr/bin/env bash
log_file=/var/log/mdlspipe.log
Working_Fullpath=$(pwd)
Working_Directory=$(basename $Working_Fullpath)
keys_of_interest=("kMDItemFSCreationDate" "kMDItemFSName")
md5_bin=$(which md5sum)
mktemp_bin=$(which mktemp)
DateTimeStamp=$(date +\%D_\%T)
IFS=$'\n'
set -a
source .env 2>&1 > /dev/null
set +a
## Functions
############
write_log () {
## Use on systems managed with systemd
# printf 'mdlspipe: %s\n' "$1" | systemd-cat
printf '%s mdlspipe: %s\n' "$DateTimeStamp" "$1" >> $log_file
}
mdls_per_file () {
file=$1
unset mdls_properties
for line in $(mdls $file); do
old_IFS=$IFS
unset IFS
mdls_key=$(printf "%s" $line | cut -d '=' -f 1)
mdls_value=$(printf "%s" $line | cut -d '=' -f 2)
if [[ " ${keys_of_interest[*]} " =~ " ${mdls_key} " ]];then
IFS=$old_IFS
mdls_properties+=( $mdls_value )
unset IFS
else
: # "Did not print!"
fi
IFS=$old_IFS
done
IFS=',';echo "${mdls_properties[*]}";IFS=$old_IFS
}
write_per_file () {
# "=== Write data to a .csv file. ===
#===================================="
# cat $temp_csv_file
# write_log "$temp_csv_file written successfully."
printf 'File: %s\nHash: %s\nMDLS Properties:\t%s\n' $1 $2 $3
}
cleanup () {
write_log "Cleanup completed successfully."
}
## Main
#######
trap cleanup EXIT
## Create temp file
temp_csv_file=$($mktemp_bin)
## Initialize column names
csv_columns=( "File Path" "File Hash" )
## Populate Columns from keys of interest
for key in ${keys_of_interest[@]};do
csv_columns+=( $key )
done
## Print columns to first line of csv
# "======= CSV Column Headers ========"
{ IFS=',';echo "${csv_columns[*]}";IFS=$'\n'
## Loop through files, writing to csv
for file in $(find $Working_Fullpath -maxdepth 1 -type f); do
## Set our hash value
hash=$($md5_bin $file | awk '{ print $4 }')
# Print csv row output
# "============= CSV Row =========="
echo "$file,$hash,$(mdls_per_file $file)"
# write_per_file $file $hash $mdls_properties
done
} >> $temp_csv_file
unset IFS
mv $temp_csv_file $PWD/mdlspipe_$(date +%\D_\%T|sed 's/[:|\/]/-/g').csv