Compare commits

..

45 Commits

Author SHA1 Message Date
Michael Holloway
b1db7f1a68 Changes to be committed:
modified:   mdlspipe
	Adding defaults to script to support bot .env and nomadic usage.
2022-03-29 22:14:29 -04:00
Michael Holloway
86a47efc6e On branch output-file
Changes to be committed:
renamed:    hash-files -> mdlspipe
2022-03-29 21:48:32 -04:00
Michael Holloway
9ffd223048 On branch docs-minimal
Changes to be committed:
new file:   LICENSE
	bsd-2clause
new file:   README.md
	Basics like summary and usage guidance.
2022-03-29 21:41:27 -04:00
Mike Holloway
b069d30fa9 On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Cleaning up comment lines
2022-03-26 19:44:40 -04:00
Mike Holloway
a889f1ff6f On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Change from two step (invoke, print) to one step (print
	invocation)
2022-03-26 19:42:50 -04:00
Mike Holloway
fb29201bcd Line 42 - Is this where the redundant mdls properties output comes from? 2022-03-26 19:36:31 -04:00
Mike Holloway
01faecccda On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Comment cleanup, output cleanup
2022-03-26 19:33:41 -04:00
Mike Holloway
ddc61edccd On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Finally, commas in-between the mdls values!
	Prepare the csv...
2022-03-26 19:28:20 -04:00
Mike Holloway
33e88fba83 modified: hash-files
Passing mdls_value to mdls_properties as a list instead of
		string variable
2022-03-26 19:26:52 -04:00
Mike Holloway
0ece9167c0 modified: hash-files 2022-03-26 19:25:59 -04:00
Mike Holloway
55be94f26a modified: hash-files 2022-03-26 19:23:47 -04:00
Mike Holloway
ac974fff80 Still troubleshooting output 2022-03-26 19:22:36 -04:00
Mike Holloway
bbc65490d3 Can we add the IFS work now? 2022-03-26 19:19:24 -04:00
Mike Holloway
f52295c6ee Adding echo in main to print rather than call.. whoops 2022-03-26 19:17:56 -04:00
Mike Holloway
320d899e95 Printing via function to debug main. 2022-03-26 19:16:36 -04:00
Mike Holloway
008de3f44c Export post-invocation of mdls_per_file function should be enough 2022-03-26 19:14:33 -04:00
Mike Holloway
652e775164 Changing where array invocation is done, still sorting out array output. 2022-03-26 19:13:14 -04:00
Mike Holloway
6a57064a74 echo vs printf for proper delimiter? Let's find out 2022-03-26 19:09:52 -04:00
Mike Holloway
fb583285e5 Even more IFS-array finagling 2022-03-26 19:07:41 -04:00
Mike Holloway
f71987d044 More IFS-in-array finagling 2022-03-26 19:06:45 -04:00
Mike Holloway
528bb7def0 IFS work for comma delimited arrays 2022-03-26 19:05:24 -04:00
Mike Holloway
16a6db49d9 whoops wrong bracket type. 2022-03-26 19:03:12 -04:00
Mike Holloway
2f5be070c8 On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Temp csv file (re)integration
2022-03-26 19:02:17 -04:00
Mike Holloway
e21989a5c3 On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	mdls_properties is accumulating values. Working on resetting
		per-file
2022-03-26 18:57:58 -04:00
Mike Holloway
60568fe7a8 Changing how mdls_properties is passed to output. 2022-03-26 18:56:41 -04:00
Mike Holloway
021d7c695a Deref'd when I should have ref'd. 2022-03-26 18:55:01 -04:00
Mike Holloway
316f57e41f On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Constructing mdls_properties array.
2022-03-26 18:54:09 -04:00
Mike Holloway
8d391800d6 Using the wrong array reference syntax? Let's find out 2022-03-26 18:49:33 -04:00
Mike Holloway
d602c76796 ....? 2022-03-26 18:48:08 -04:00
Mike Holloway
dbfe55872f Do I have it? 2022-03-26 18:47:02 -04:00
Mike Holloway
1fd34742fe Still getting duplicate keys. Investigating. 2022-03-26 18:45:56 -04:00
Mike Holloway
3ac7a29917 Teasing out duplicate key... 2022-03-26 18:43:44 -04:00
Mike Holloway
018d832456 Further clarification of mdls_properties list. 2022-03-26 18:41:50 -04:00
Mike Holloway
7d82f754db Debugging duplicate MDLS Column names 2022-03-26 18:40:31 -04:00
Mike Holloway
6f6fedcb95 Reintroducing filter 2022-03-26 18:37:40 -04:00
Mike Holloway
191d690603 Investigating main for loop and evocation of mdls_per_file 2022-03-26 18:34:55 -04:00
Mike Holloway
71e228c6c2 Iteration errors? Getting duplicate MDLS property column names? 2022-03-26 18:33:22 -04:00
Mike Holloway
2a491037a0 Improving debug output for mdls values as prepared for csv lines 2022-03-26 18:30:54 -04:00
Mike Holloway
05c46480fa On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Important to note - macOS `md5` the checksum is field 4
		whereas Linux `md5sum` the checksum is field 1
2022-03-26 18:28:28 -04:00
Mike Holloway
329f3a71ff On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Select the correct md5sum field with awk.
2022-03-26 18:27:17 -04:00
Mike Holloway
986aada28f Passing arguments to write_per_file 2022-03-26 18:26:02 -04:00
Mike Holloway
30d39a1060 Whoops, then I forgot I need dummy output for write_per_file! 2022-03-26 18:23:37 -04:00
Mike Holloway
bbc0007917 Whoops, empty function. 2022-03-26 18:22:02 -04:00
Mike Holloway
c9d13e3a25 On branch rewrite-2022-03-26
Your branch is up to date with 'origin/rewrite-2022-03-26'.

 Changes to be committed:
modified:   hash-files
	Moving csv file preparation outside per-file loop.
2022-03-26 18:21:04 -04:00
Mike Holloway
f32e7b9242 On branch rewrite-2022-03-26
Changes to be committed:
modified:   hash-files
	Removing functions, refactoring, testing
2022-03-26 18:16:38 -04:00
4 changed files with 116 additions and 118 deletions

8
LICENSE Normal file
View File

@ -0,0 +1,8 @@
Copyright (c) Michael Holloway, 2022
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

21
README.md Normal file
View File

@ -0,0 +1,21 @@
mdlspipe
========
v0.1.1
Supported Platforms: Mac OS X
Provided a directory (default is current directory), capture particular
fields of `mdls` output for each file in provided directory in a csv.
Usage
-----
Run directly:
./mdlspipe
Place mdlspipe in your $PATH and run from $PWD.
By default, mdlspipe will write the output data to a csv called
'mdlspipe_YYYY-MM-DD_HH-MM-SS.csv'.

View File

@ -1,118 +0,0 @@
#!/usr/bin/env bash
log_file=/var/log/hash-files.log
keys_of_interest=("kMDItemFSCreationDate" "kMDItemFSName")
md5_bin=$(which md5sum)
mktemp_bin=$(which mktemp)
DateTimeStamp=$(date +\%D_\%T)
IFS=$'\n'
set -a
source .env 2>&1 > /dev/null
set +a
## Functions
############
write_log () {
#echo " ++ Write to log.
# +++++++++++++++"
## Use on systems managed with systemd
# printf 'hash-files: %s\n' "$1" | systemd-cat
printf '%s hash-files: %s\n' "$DateTimeStamp" "$1" >> $log_file
}
create_list_of_files () {
#echo "=== Get list of files in $Working_Directory. ===
#=================================================="
for file in $(find $Working_Fullpath -maxdepth 1 -type f); do
list_of_files+=( $file )
done
write_log "List of files created successfully."
}
mdls_files () {
for file in $list_of_files; do
for line in $(mdls $file); do
old_IFS=$IFS
unset IFS
mdls_key=$(printf "%s" $line | cut -d '=' -f 1)
mdls_value=$(printf "%s" $line | cut -d '=' -f 2)
if [[ " ${keys_of_interest[*]} " =~ " ${mdls_key} " ]];then
IFS=$old_IFS
: #printf "\n============================\nMDLS Key: %s\nMDLS Value: %s\n============================\n\n" $mdls_key $mdls_value
unset IFS
else
: #echo "Did not print!"
fi
IFS=$old_IFS
done
done
}
hash_files () {
#echo "=== Hash files. ===
#======================"
for file in ${list_of_files[@]}; do
hash=$($md5_bin $file | awk '{ print $1 }')
list_of_hashes+=( $hash )
done
write_log "Files hashed successfully."
}
#create_csv () {
#echo "=== Create array of comma-separated-values. ===
#=================================================="
# for i in ${!list_of_files[@]}; do
# csv_array+=( ${list_of_files[i]},${list_of_hashes[i]} )
# done
# export csv_array
# write_log "CSV Array creation successful."
#}
write_data () {
#echo "=== Write data to a .csv file. ===
#===================================="
## Create temp file
temp_csv_file=$($mktemp_bin)
## Initialize column names
csv_columns=( "File Path" "File Hash" )
## Populate Columns from keys of interest
for key in ${keys_of_interest[@]};do
csv_columns+=( ${keys_of_interest[key]} )
done
## Print columns to first line of csv
IFS=',';echo "${csv_columns[*]}";IFS=$'\n' >> $temp_csv_file
## Print csv array rows following columns row
for row in ${csv_array[@]};do
printf '%s\n' $row
done >> $temp_csv_file
# cat $temp_csv_file
write_log "$temp_csv_file written successfully."
}
cleanup () {
write_log "Cleanup completed successfully."
}
## Main
#######
trap cleanup EXIT
echo "Main Start"
create_list_of_files
mdls_files
hash_files
if [ ${#list_of_files[@]} != ${#list_of_hashes[@]} ];then
echo "Error! Number of files and hashes differs. Exiting..." && exit
else
: #create_csv
fi
#write_data
#while $list_of_files >=1; do
#done
unset IFS

87
mdlspipe Executable file
View File

@ -0,0 +1,87 @@
#!/usr/bin/env bash
log_file=/var/log/mdlspipe.log
Working_Fullpath=$(pwd)
Working_Directory=$(basename $Working_Fullpath)
keys_of_interest=("kMDItemFSCreationDate" "kMDItemFSName")
md5_bin=$(which md5sum)
mktemp_bin=$(which mktemp)
DateTimeStamp=$(date +\%D_\%T)
IFS=$'\n'
set -a
source .env 2>&1 > /dev/null
set +a
## Functions
############
write_log () {
## Use on systems managed with systemd
# printf 'mdlspipe: %s\n' "$1" | systemd-cat
printf '%s mdlspipe: %s\n' "$DateTimeStamp" "$1" >> $log_file
}
mdls_per_file () {
file=$1
unset mdls_properties
for line in $(mdls $file); do
old_IFS=$IFS
unset IFS
mdls_key=$(printf "%s" $line | cut -d '=' -f 1)
mdls_value=$(printf "%s" $line | cut -d '=' -f 2)
if [[ " ${keys_of_interest[*]} " =~ " ${mdls_key} " ]];then
IFS=$old_IFS
mdls_properties+=( $mdls_value )
unset IFS
else
: # "Did not print!"
fi
IFS=$old_IFS
done
IFS=',';echo "${mdls_properties[*]}";IFS=$old_IFS
}
write_per_file () {
# "=== Write data to a .csv file. ===
#===================================="
# cat $temp_csv_file
# write_log "$temp_csv_file written successfully."
printf 'File: %s\nHash: %s\nMDLS Properties:\t%s\n' $1 $2 $3
}
cleanup () {
write_log "Cleanup completed successfully."
}
## Main
#######
trap cleanup EXIT
## Create temp file
temp_csv_file=$($mktemp_bin)
## Initialize column names
csv_columns=( "File Path" "File Hash" )
## Populate Columns from keys of interest
for key in ${keys_of_interest[@]};do
csv_columns+=( $key )
done
## Print columns to first line of csv
# "======= CSV Column Headers ========"
{ IFS=',';echo "${csv_columns[*]}";IFS=$'\n'
## Loop through files, writing to csv
for file in $(find $Working_Fullpath -maxdepth 1 -type f); do
## Set our hash value
hash=$($md5_bin $file | awk '{ print $4 }')
# Print csv row output
# "============= CSV Row =========="
echo "$file,$hash,$(mdls_per_file $file)"
# write_per_file $file $hash $mdls_properties
done
} >> $temp_csv_file
unset IFS
mv $temp_csv_file $PWD/mdlspipe_$(date +%\D_\%T|sed 's/[:|\/]/-/g').csv