shell2
no way to compare when less than two revisions
Differences
This shows you the differences between two versions of the page.
Next revision | |||
— | shell2 [2019/12/17 16:40] – search is hanging on shell page. This is a test page. dblume | ||
---|---|---|---|
Line 1: | Line 1: | ||
+ | ====== shell tips ====== | ||
+ | Here's a tutorial: [[http:// | ||
+ | |||
+ | ===== Quick Tips ===== | ||
+ | |||
+ | The [[http:// | ||
+ | |||
+ | <code bash> | ||
+ | $ !?scp?:p | ||
+ | $ ^rscp | ||
+ | </ | ||
+ | |||
+ | ===== bash expansion ===== | ||
+ | |||
+ | <code bash> | ||
+ | $ cp file{,.bk} | ||
+ | </ | ||
+ | |||
+ | expands to | ||
+ | |||
+ | <code bash> | ||
+ | $ cp file file.bk | ||
+ | </ | ||
+ | |||
+ | Replace all files that end with .JPG to .jpeg | ||
+ | |||
+ | <code bash> | ||
+ | for file in *.JPG; do mv $file ${file%.JPG}.jpeg; | ||
+ | for file in *.JPG; do mv $file ${file/ | ||
+ | </ | ||
+ | |||
+ | Then there are two different " | ||
+ | |||
+ | <code bash> | ||
+ | rename .JPG .jpg *.JPG | ||
+ | rename " | ||
+ | </ | ||
+ | |||
+ | |||
+ | ===== Command Template ===== | ||
+ | |||
+ | Here's a template for shell commands that demonstrates a number of arguments, length of argument, etc. It could still stand a bit of clean-up according to the [[http:// | ||
+ | |||
+ | Another good resource is [[http:// | ||
+ | |||
+ | <code bash> | ||
+ | # | ||
+ | set -eu -o pipefail # See: https:// | ||
+ | |||
+ | declare -r SCRIPT_NAME=$(basename " | ||
+ | |||
+ | ## exit the shell (default status code: 1) after printing the message to stderr | ||
+ | die() { | ||
+ | echo >&2 " | ||
+ | exit ${2-1} | ||
+ | } | ||
+ | |||
+ | ## the options used by this script | ||
+ | DISK=e | ||
+ | declare -i VERBOSE=0 | ||
+ | |||
+ | ## exit the shell (with status 2) after printing the message | ||
+ | usage() { | ||
+ | echo "\ | ||
+ | $SCRIPT_NAME -hv [Drive Letter] (default: $DISK) | ||
+ | -h Print this help text | ||
+ | -v Enable verbose output | ||
+ | " | ||
+ | exit 2; | ||
+ | } | ||
+ | |||
+ | ## Process the options | ||
+ | while getopts " | ||
+ | do | ||
+ | case $OPTION in | ||
+ | h) usage;; | ||
+ | v) VERBOSE=1;; | ||
+ | \?) usage;; | ||
+ | esac | ||
+ | done | ||
+ | |||
+ | ## Process the arguments | ||
+ | shift $(($OPTIND - 1)) | ||
+ | |||
+ | if [ $# -eq 0 ]; then | ||
+ | : # Let the default be used | ||
+ | elif [ $# -eq 1 ]; then | ||
+ | if [ ${#1} -eq 1 ]; then | ||
+ | DISK=$1 | ||
+ | else | ||
+ | # 64 is EX_USAGE from sysexits.h | ||
+ | die " | ||
+ | fi | ||
+ | else | ||
+ | usage; | ||
+ | fi | ||
+ | |||
+ | ## Lock this if only one instance can run at a time | ||
+ | # UNIQUE_BASE=${TMPDIR: | ||
+ | LOCK_FILE=${TMPDIR: | ||
+ | if [ -f " | ||
+ | die " | ||
+ | fi | ||
+ | trap "rm -f $LOCK_FILE" | ||
+ | touch $LOCK_FILE | ||
+ | |||
+ | ## The main work of this script | ||
+ | |||
+ | if [ ! -d / | ||
+ | mkdir -p / | ||
+ | fi | ||
+ | |||
+ | ((VERBOSE==1)) && echo " | ||
+ | rsync / | ||
+ | |||
+ | # We add "|| true" because we don't want to stop | ||
+ | # if the directory was already empty | ||
+ | rm -r / | ||
+ | |||
+ | # Note how we find the number of cores to use | ||
+ | make -C build_subdirectory all -j$(grep -c ^processor / | ||
+ | </ | ||
+ | |||
+ | ===== Miscellaneous Shell Tips ===== | ||
+ | |||
+ | If you want a single column of just the file and path names, you can get it like so: | ||
+ | |||
+ | <code bash> | ||
+ | ls --format=single-column | ||
+ | </ | ||
+ | |||
+ | But if you don't know what you're doing, you might construct something like so: | ||
+ | |||
+ | <code bash> | ||
+ | ls -Al | tr -s ' ' | cut -d ' ' -f10- | ||
+ | </ | ||
+ | |||
+ | - List " | ||
+ | - Squeeze repeats of the space character | ||
+ | - Cut out everything from before the 10th column and show everything afterwards. | ||
+ | |||
+ | Of course, if you could assert the following: | ||
+ | |||
+ | * none of the first columns were repeats (awk would only identify the first repeated column) | ||
+ | * the desired column didn't have delimiters in it (filenames with spaces) | ||
+ | |||
+ | ...you could use awk | ||
+ | |||
+ | <code bash> | ||
+ | ... | awk ' | ||
+ | </ | ||
+ | |||
+ | Anyway, given a list of directories, | ||
+ | |||
+ | <code bash> | ||
+ | cat list_of_directories_at_one_level.txt | xargs -I {} cp -r $SOURCEDIRPREFIX: | ||
+ | </ | ||
+ | |||
+ | Useful bash command for finding strings within python files... | ||
+ | |||
+ | <code bash> | ||
+ | find . -name \*.py -type f -print0 | xargs -0 grep -nI " | ||
+ | find . -type f \( -name \*.[ch]pp -or -name \*.[ch] \) -print0 | xargs -0 grep -nI printf | ||
+ | </ | ||
+ | |||
+ | Interesting way to use '' | ||
+ | |||
+ | <code bash> | ||
+ | #!/bin/bash | ||
+ | find $PWD -regex " | ||
+ | grep -v " \|unwantedpath/ | ||
+ | cscope -q -b | ||
+ | </ | ||
+ | |||
+ | Here's how to find if a symbol is in a library, and how to search lots of object files and print the filename above the search... | ||
+ | |||
+ | <code bash> | ||
+ | nm obj-directory/ | ||
+ | find bindirectory/ | ||
+ | c++filt | grep -P " | ||
+ | </ | ||
+ | |||
+ | Also handy to merge two streams together... | ||
+ | |||
+ | <code bash> | ||
+ | ( cat file1 && cat file2 ) | sort | ||
+ | </ | ||
+ | |||
+ | When a little quick math is needed, use '' | ||
+ | <code bash> | ||
+ | $ bc <<< | ||
+ | F | ||
+ | $ bc -l <<< | ||
+ | .33333333333333333333 | ||
+ | $ bc <<< | ||
+ | .33 | ||
+ | $ bc <<< | ||
+ | 11 | ||
+ | </ | ||
+ | |||
+ | and, when coverting from hex to dec... | ||
+ | |||
+ | <code bash> | ||
+ | echo $((0x2dec)) | ||
+ | </ | ||
+ | |||
+ | But, then again, does that really seem easier than, | ||
+ | |||
+ | < | ||
+ | python -c "print int(' | ||
+ | </ | ||
+ | |||
+ | There' | ||
+ | |||
+ | <code bash> | ||
+ | $ echo $(( ($(date +%s) - $(date -d " | ||
+ | </ | ||
+ | |||
+ | And a Python way... | ||
+ | |||
+ | <code python> | ||
+ | python -c " | ||
+ | </ | ||
+ | |||
+ | And for displaying lines clipped at the right edge of the window instead wrapped: | ||
+ | |||
+ | <code bash> | ||
+ | cat_one_line_per_row() { | ||
+ | cat " | ||
+ | } | ||
+ | </ | ||
+ | |||
+ | or a " | ||
+ | |||
+ | <code bash> | ||
+ | alias clip=" | ||
+ | </ | ||
+ | |||
+ | ctags' | ||
+ | |||
+ | <code bash> | ||
+ | ctags -n --if0=yes --c++-kinds=+p --langmap=c++: | ||
+ | --langmap=asm: | ||
+ | --exclude=unwanted_file.lst \ | ||
+ | --exclude=' | ||
+ | --regex-C++='/ | ||
+ | </ | ||
+ | |||
+ | When you want to repeat a command a few times... | ||
+ | <code bash> | ||
+ | seq 1 50 | xargs -I{} -n1 echo '{} Hello World!' | ||
+ | </ | ||
+ | |||
+ | When you've set up Perforce to use an application for diff with '' | ||
+ | |||
+ | <code bash> | ||
+ | $ P4DIFF=; p4 diff hello-world.cpp | ||
+ | </ | ||
+ | |||
+ | It's [[http:// | ||
+ | |||
+ | So, use '' | ||
+ | |||
+ | <code bash> | ||
+ | p4_sync() { | ||
+ | p4 changes -s submitted -m1 ... | tee p4_sync_to_change.txt | ||
+ | changelist=`cut -d " " -f 2 p4_sync_to_change.txt` | ||
+ | changelist_filename=changelist.h | ||
+ | p4 sync ...@$changelist | ||
+ | if [ -w $changelist_filename ] | ||
+ | then | ||
+ | sed -i ' | ||
+ | fi | ||
+ | } | ||
+ | </ | ||
+ | |||
+ | Note the use of $@ vs " | ||
+ | |||
+ | <code bash> | ||
+ | telnet_log() { | ||
+ | curtime=$(date -Iseconds | tr : .) | ||
+ | args=$(echo " | ||
+ | telnet $@ | tee $HOME/ | ||
+ | } | ||
+ | |||
+ | last_telnet_log() { | ||
+ | ls -d1t $HOME/ | ||
+ | } | ||
+ | </ | ||
+ | |||
+ | Of course if you do that, you'll want to occasionally (via cronjob?) delete old archives. | ||
+ | |||
+ | < | ||
+ | find $HOME/ | ||
+ | </ | ||
+ | |||
+ | Keywords: bash shell sh zsh | ||
+ | |||
+ | ====== .vimrc tips ====== | ||
+ | |||
+ | Here's an alternative way to automatically save backups (with dates in the filename) everytime you save a file. | ||
+ | < | ||
+ | set backup | ||
+ | set backupdir=~/ | ||
+ | au BufWritePre * let &bex = ' | ||
+ | </ | ||
+ | |||
+ | That makes a lot of files, so you can clean out the backups with a cron job like this: | ||
+ | |||
+ | < | ||
+ | # at 3 in the morning on Mondays, delete files older than 30 days | ||
+ | 0 3 * * 1 find $HOME/ | ||
+ | </ | ||
+ | |||
+ | ====== expect tips ====== | ||
+ | |||
+ | What to do when it's not sure you're going to make a connection? | ||
+ | |||
+ | < | ||
+ | set times 0 | ||
+ | set made_connection 0 | ||
+ | set timeout 120 | ||
+ | while { $times < 2 && $made_connection == 0 } { | ||
+ | spawn nc $SERVER | ||
+ | send " | ||
+ | expect { | ||
+ | " | ||
+ | send " | ||
+ | set made_connection 1 | ||
+ | } eof { | ||
+ | sleep 1s | ||
+ | set times [ expr $times + 1 ] | ||
+ | } timeout { | ||
+ | puts " | ||
+ | exit | ||
+ | } | ||
+ | } | ||
+ | } | ||
+ | </ | ||
+ | |||
+ | I think the following is wrong-headed. It's not usually the case that spawn will fail. | ||
+ | |||
+ | < | ||
+ | set times 0; | ||
+ | while { $times < 2 && $made_connection == 0 } { | ||
+ | if { [ catch { spawn nc $SERVER } pid ] } { | ||
+ | set times [ expr $times + 1 ]; | ||
+ | sleep 1s; | ||
+ | } else { | ||
+ | set made_connection 1 | ||
+ | } | ||
+ | } | ||
+ | </ | ||
+ | |||
+ | ====== Perl tips ====== | ||
+ | |||
+ | The module '' | ||
+ | |||
+ | ====== Application Memory Usage ====== | ||
+ | |||
+ | Use VM Resident Set Size. See VmRSS below. (Note the [[http:// | ||
+ | |||
+ | <code bash> | ||
+ | host:# ps -ef | grep etflix | ||
+ | default | ||
+ | root 2108 1046 0 22:26 ? 00:00:00 grep etflix | ||
+ | host:# pidof netflix | ||
+ | 1532 | ||
+ | host:# cat / | ||
+ | Name: MAIN | ||
+ | ... | ||
+ | Groups: | ||
+ | VmPeak: | ||
+ | VmSize: | ||
+ | VmLck: | ||
+ | VmHWM: | ||
+ | VmRSS: | ||
+ | ... | ||
+ | </ | ||
+ | |||
+ | Or, while running an application, | ||
+ | <code bash> | ||
+ | while [ 1 ] | ||
+ | do | ||
+ | free -m | grep Mem | ||
+ | sleep 3 | ||
+ | done | ||
+ | </ | ||
+ | Alternatively, | ||
+ | <code bash> | ||
+ | while true; do sync; cat / | ||
+ | </ | ||
+ | ====== Measuring Available Memory ====== | ||
+ | |||
+ | This note doesn' | ||
+ | |||
+ | The best measure I've found for " | ||
+ | |||
+ | The command '' | ||
+ | ====== Shared Memory Usage ====== | ||
+ | |||
+ | To increase limit to 256MB from command line: | ||
+ | |||
+ | <code bash> | ||
+ | echo " | ||
+ | echo " | ||
+ | </ | ||
+ | |||
+ | Or, edit / | ||
+ | |||
+ | < | ||
+ | kernel.shmmax= 268435456 | ||
+ | kernel.shmall= 268435456 | ||
+ | </ | ||
+ | |||
+ | ====== Performance Metrics ====== | ||
+ | |||
+ | * Use [[http:// | ||
+ | * [[https:// | ||
+ | |||
+ | And you can scrape logs that start with timecodes to create Spreadsheet charts. Given logs like: | ||
+ | |||
+ | < | ||
+ | 2016-10-13 19: | ||
+ | </ | ||
+ | |||
+ | On a Macintosh: | ||
+ | |||
+ | <code bash> | ||
+ | grep memory devicelogs.txt | tr -s ' ' | cut -d " " -f 1,2,4 | \ | ||
+ | sed ' | ||
+ | open heapinfo.csv -a " | ||
+ | </ | ||
+ | |||
+ | And on Linux, instead of opening Microsoft Excel, that last line would be: | ||
+ | |||
+ | <code bash> | ||
+ | libreoffice --calc heapinfo.csv | ||
+ | </ | ||
+ | |||
+ | ====== Cron ====== | ||
+ | |||
+ | Keep tasks serialized with [[https:// | ||
+ | |||
+ | ( | ||
+ | flock -n 9 || exit 1 | ||
+ | # ... commands executed under lock ... | ||
+ | ) 9>/ | ||
+ | |||
+ | |||
+ | ====== Retrieving Symbols with addr2line ====== | ||
+ | |||
+ | You can gather a backtrace (stacktrace) with this piped command to addr2line. | ||
+ | |||
+ | $ cat << EOF | cut -d " " -f 3 | tr -d " | ||
+ | addr2line -e builds/ | ||
+ | xargs -d ' | ||
+ | > 7/22 app() [0xf7878] (0xf7878) | ||
+ | > 8/22 app() [0x39c2f8] (0x39c2f8) | ||
+ | > 9/22 app() [0xe1964] (0xe1964) | ||
+ | > EOF | ||
+ | src/ | ||
+ | src/ | ||
+ | src/ | ||
+ | | ||
+ | ====== Sort by Frequency ====== | ||
+ | |||
+ | I ran the following P4 command to find out who's been editing a file recently: | ||
+ | |||
+ | $ find . -name fname.cpp | xargs p4 filelog -s -m 10 | \ | ||
+ | awk '/ | ||
+ | |||
+ | |||
+ | ====== jq Tips ====== | ||
+ | |||
+ | jq is really handy. Here's a tip for some processing I often do: | ||
+ | |||
+ | <file javascript fruits.txt> | ||
+ | { " | ||
+ | { | ||
+ | " | ||
+ | { | ||
+ | " | ||
+ | " | ||
+ | }, | ||
+ | " | ||
+ | { | ||
+ | " | ||
+ | " | ||
+ | }, | ||
+ | " | ||
+ | " | ||
+ | } | ||
+ | } | ||
+ | </ | ||
+ | |||
+ | < | ||
+ | $ jq ' | ||
+ | { | ||
+ | " | ||
+ | " | ||
+ | } | ||
+ | |||
+ | # with_entries(f) is an alias for to_entries | map(x) | from_entries | ||
+ | |||
+ | jq ' | ||
+ | { | ||
+ | " | ||
+ | " | ||
+ | } | ||
+ | |||
+ | $ jq ' | ||
+ | { | ||
+ | " | ||
+ | " | ||
+ | } | ||
+ | </ | ||
+ | |||
+ | <file javascript fruit_ip.txt> | ||
+ | { | ||
+ | " | ||
+ | " | ||
+ | " | ||
+ | " | ||
+ | " | ||
+ | } | ||
+ | }, | ||
+ | " | ||
+ | " | ||
+ | " | ||
+ | " | ||
+ | " | ||
+ | } | ||
+ | } | ||
+ | } | ||
+ | </ | ||
+ | |||
+ | < | ||
+ | $ jq ' | ||
+ | { | ||
+ | " | ||
+ | " | ||
+ | } | ||
+ | |||
+ | $ jq ' | ||
+ | { | ||
+ | " | ||
+ | " | ||
+ | } | ||
+ | |||
+ | $ jq -r " | ||
+ | David' | ||
+ | David' | ||
+ | </ | ||
+ | |||
+ | ====== Protips for find ====== | ||
+ | |||
+ | [[https:// | ||
+ | |||
+ | Given a lib directory, I wanted to find all the actual .so files that needed libz. | ||
+ | |||
+ | find lib -type f -name \*.so\* -exec sh -c ' | ||
+ | | ||
+ | Note that you can pass -print (or -and -print) after a -exec argument. Also, the " - " is just a placeholder for $0 (usually the command name, in this case " | ||
+ | |||
+ | NEEDED | ||
+ | lib/ | ||
+ | |||
+ | ====== Additional Keywords ====== | ||
+ | |||
+ | Linux, Unix, *nix |
shell2.txt · Last modified: 2023/05/17 09:47 by dblume