#!/bin/ksh
#       
#       Copyright (c) 1996-2004 Silicon Graphics, Inc.
#       All rights reserved.
#
#               run_merge_mgr.sh
#
# This script runs merges for VGs within a specific DG, taking into account
# the availability of tape drives, and the differing requirements of the VGs
# for more tapes, but with minimal impact on the existing workload.
#
# It runs from a task associated with a DG in a leisurely loop until it
# can't find any work to do, or run_merge_stop is run.  It's associated with
# a specific DG because tape drives are the bottleneck.  Sure, cache disk is
# another, but that can easily be handled by scheduling the various DG-based
# invocations at different times.  We may be moving to an environment where
# many VGs might be in use, but the number of DGs will still be modest.
#
#
# The first thing the MM does is figure out the number of idle drives are
# available in the DG.  Using a load-average sort of algorithm, it is
# cautious in accepting an increase in the number of idle drives, but reacts
# right away to a decrease.  Obviously, this is to minimise the impact on
# the normal workload.
#
# If the number of drives is > 0 (> 1 if CACHE_SPACE is enforcing socket
# merges), then it examines the needs of all the VGs in that DG.  It
# calculates the number of effective tapes the VG has available to write
# on.  This is the number of empty, HFREE, or HSPARSE tapes, plus half the
# number of partial tapes (because on average, they're half-full), plus a
# square-root-based share of any associated AG, moderated by the value of
# ALLOCATION_MAXIMUM.  This is the same algorithm used by the VLM and the
# RW, so they should all be consistent.
#
# The number of effective tapes is biased by the value of MIN_VOLUMES,
# because that is considered to be the expression of activity according to
# the administrator.  For example, a VG with 5 effective tapes but a
# MIN_VOLUMES of 10 is just as desperate for more tapes as another VG with
# values of 50 and 100 resp.
#
# The VGs are sorted by this biased number of effective tapes, and the the
# MM tries to hsparse a certain number of tapes for the one most desperate
# VG.  "Certain number" depends on the normal THRESHOLD, VOLUME_LIMIT and
# DATA_LIMIT merge parameters, and is subject to a ceiling equal to the
# number of idle drives (less 1 if socket merges are being enforced).
#
#     Special case: if the number of effective tapes is less than
#     MIN_VOLUMES, the value of THRESHOLD is doubled (but capped at 99%)
#     because such a VG is *truly* desperate for tapes, and THRESHOLD should
#     be eased off a bit.
#
# Obviously, if there are plenty of merge candidates, the tapes selected are
# those with the least amount of data remaining, because they give the most
# benefit with the least effort.
#
# If the "certain number" happens to be zero, then the process is repeated
# for the second most desperate VG, and if necessary the third, and so on.
# Merges are confined to a single VG in each pass through the loop, to
# maximise the chances of many-to-one merges, where the number of input
# tapes is greater than the number of output ones.  If no VG has any
# selectable tapes, then the MM exits.
#
# Otherwise, the MM sleeps for several minutes, without waiting for the
# merges to complete, and loops back and does it all again, probably
# selecting a different VG this time.
#
#
# All of this feeds off the text files produced by the Resource Watcher.  A
# few bits of information are not there, and have to be found with
# dmvoladm.
#
#
# It is invoked by dmatls as requested by the site via a RUN_TASK
# parameter in a drivegroup object.  It is not for use by any of the MSPs.
#
# USAGE:
#
#       RUN_TASK        $ADMINDIR/run_merge_mgr.sh every day at 23:00
#
# Script is invoked as:
#       run_merge_mgr ls_name task_group_name dg_name
#
# where the taskgroup is referred to from the DG stanza, and uses the same
# parameters as run_tape_merge.sh, namely THRESHOLD, VOLUME_LIMIT, DATA_LIMIT
# and RUN_TASK.
#
# The LS to which this DG belongs must have a RW configured.

DEBUG=0			# 1 turns on debugging messages
			# 2 suppresses DB changes

CYCLE_TIME=2		# Number of mins to sleep between attempts to find work
			# It should not be too small, or it'll be acting again
			# before its previous actions are reflected in the RW.
LOAD_FN_PERCENT=25	# Degrade previous load-averaged number of drives
			# before adding in the newest value (a percentage)

RW_VERSION=1.0		# Latest version of RW format this script can handle
			# That is, it needs to be reverified at 2.0 and above,
			# and it doesn't require anything added after 1.0.

# Invoke common startup dot-script

PATH=/usr/lib/dmf		# only for locating the dot-file
TMP=				# make sure we use a private TMP directory
. dmf_script_startup.dot

USAGE="Usage: $MYSELF ls-name taskgroup-name dg-name"

# Check that we are being called by a DG with a RW

_assert_root
if [ $# -ne 3 ]; then
    _usage
fi
RW=$(dmconfig $LS WATCHER 2>/dev/null | awk '{print $1; exit}')
if [[ -z $RW ]]; then
    _log E \
	$MYSELF must be called from a DG whose LS has a RW configured - aborting
    exit 1
fi

last_update='unknown'
starting_up=1
typeset -i drives_idle_by_100=50
print $$ >| $TMP/$MYSELF.pid
_log O Looking for mergeable tapes within DG $DG

while [ -r $TMP/$MYSELF.pid ]; do
    _assert_dmf_up

    # The extraction of config details is inside the loop, to allow for
    # changing them on the fly

    RW_DIR=$(dmconfig -p $LS SPOOL_DIR)/_$RW
    if [ ! -d $RW_DIR ]; then
	log E $RW_DIR does not exist - aborting
	exit 1
    fi

    errors=''
    if ! THRESHOLD=$(dmconfig $TASKGROUP THRESHOLD 2>/dev/null); then
	_log E Bad value for THRESHOLD for $TASKGROUP
	errors="THRESHOLD $errors"
    fi
    if ! VOLUME_LIMIT=$(dmconfig $TASKGROUP VOLUME_LIMIT 2>/dev/null); then
	_log E Bad value for VOLUME_LIMIT for $TASKGROUP
	errors="VOLUME_LIMIT $errors"
    fi
    if ! DATA_LIMIT=$(dmconfig $TASKGROUP DATA_LIMIT 2>/dev/null); then
	_log E Bad value for DATA_LIMIT for $TASKGROUP
	errors="DATA_LIMIT $errors"
    fi
    DATA_LIMIT_EXP=$(dmconfig -n $TASKGROUP DATA_LIMIT 2>/dev/null)
    if [[ -n "$errors" ]]; then
	(print "The following parameters of $TASKGROUP are invalid:";
        PARAMS=
	paramslist=
	for v in $errors; do
	    print "\t$v"
	    if [[ -z $PARAMS ]]; then
	    	PARAMS=$v
		paramslist=$v
	    else
		PARAMS="$PARAMS;$v"
		paramslist="$paramslist, $v"
	    fi;
	done;
	print "\n$MYSELF task not run.") |
	    _send_mail "Error in configuration of $TASKGROUP on $HOST"
	if [[ -f $POSTDMALERT ]]; then
	    $POSTDMALERT -p 2 -c RUNMERGEMGR001 -s task -k \
	    "cmd:$THISCMD,taskgroup:$TASKGROUP,param:$PARAMS" \
	    "$MYSELF: Error in configuration of $TASKGROUP on $HOST." \
	    "The following parameters of $TASKGROUP are invalid: $paramslist." \
	    >/dev/null 2>&1
	fi
	exit 1
    fi
    VOLUME_GROUPS=$(dmconfig $DG VOLUME_GROUPS 2>/dev/null)
    MESSAGE_LEVEL=$(dmconfig $TASKGROUP MESSAGE_LEVEL 2>/dev/null)
    if (( ${MESSAGE_LEVEL:=2} < -1 )); then
	MESSAGE_LEVEL='-1'
    fi
    if (( MESSAGE_LEVEL > 2 )); then
	MESSAGE_LEVEL=2
    fi

    if [ $(cat $TMP/$MYSELF.pid) != $$ ]; then
	_log O "Another $MYSELF (PID $(cat $TMP/$MYSELF.pid)) is running" \
		"- closing down"
	break
    fi

     print $VOLUME_GROUPS \
     | awk '
	#   Global Tables:
	#     ag_refs[agname]		number of VGs using the AG
	#     ags[agname]		number of tapes in the AG
	#     vg_ag[vgname]		name of the AG, or ""
	#     vg_allowed[vgname]	maximum number of drives allowed
	#     vg_assigned[vgname]	number of drives in use
	#     vg_empty[vgname]		number of empty tapes
	#     vg_in_ag[vgname]	(NIU)	number of tapes in the AG, or "N/A"
	#     vg_minimum[vgname]	minimum number of effective tapes
	#     vg_partial[vgname]	number of partial tapes
	#     vg_score_val[vgname]   \_	linked list of need for more tapes
	#     vg_score_nxt[vgname]   /	    (less is more desperate)
	#     vg_sparse[vgname]		number of HSPARSEd tapes
	#     vgs[vgname] 		placeholder

	function _continue_after_exit() {
	    _debug(V, "\t\t-- will iterate again after a sleep")
	    skip_end_processing = 1
	    exit 0
	}

	function _break_after_exit() {
	    _debug(V, "\t\t-- will terminate")
	    skip_end_processing = 1
	    exit 1
	}

	function _min(_a, _b) {
	    return _a < _b ? _a : _b
	}

	function _max(_a, _b) {
	    return _a > _b ? _a : _b
	}

	function _setenv(_vble, _value) {
	    if (_value == "") {
		_log(E, "Warning - setting " _vble " to null")
	    }
	    print "export " _vble "=\"" _value "\""
	}

	function update_time() {
	    date | getline now
	    close(date)
	}

	function _log(_level, _text) {
	    if (!_text) {
		print("???? bad call to _log \"" _level "\"") | stderr
		_break_after_exit()
	    }
	    if (msg_level[_level] <= MESSAGE_LEVEL) {
		printf("%s-%c %8s %8d-%s %s\n", \
			now, _level, hostname, PID, MYSELF, _text) \
		| stderr
	    }
	}

	function _debug(_level, _text) {
	    if (!_text) {
		_log(E, "???? bad call to _debug \"" _level "\"")
		_break_after_exit()
	    }
	    if (DEBUG) {
		if (_level == E) {
		    _log(_level, "???? " _text)
		} else {
		    _log(_level, "?? " _text)
		}
	    }
	}

	function _dmconfig(_args,		_cmd, _result) {
	    _cmd = "dmconfig " _args " 2>/dev/null"
	    _cmd | getline _result
	    close(_cmd)
	    return _result
	}

	function _find_best_vg(			_dmvoladm, _ag, _vg, _score,
						_from_ag, _effective_volumes,
						_max_volumes, _p, _n) {
	    _debug(V, "\t\t-- Entering _find_best_vg()")
	    vg_score_list = ""
	    for (_ag in ags) {
		if (ag_refs[_ag]) {
		    _debug(I, "AG " _ag " has " ags[_ag] " tapes and " \
			    ag_refs[_ag] " refs")
		}
	    }
	    for (_vg in vgs) {
		_debug(O, _vg \
		    "\tminimum=" vg_minimum[_vg] \
		    "  sparse=" vg_sparse[_vg] \
		    "  partial=" vg_partial[_vg] \
		    "  empty=" vg_empty[_vg] \
		    "  ag_max=" vg_ag_max[_vg] \
		    "  ag=\"" vg_ag[_vg] "\"")

		# Figure out the number of tapes available for writing
		_max_volumes = _effective_volumes = vg_empty[_vg]

		# Partial tapes are half full on average
		_effective_volumes += int(vg_partial[_vg] / 2)
		_max_volumes += vg_partial[_vg]

		# Add in currently hsparse tapes - they will become available
		# in time (ignoring the possibility that they might transfer
		# to an AG when that flag is cleared)
		_effective_volumes += vg_sparse[_vg]

		# Add in currently hfree tapes - they will become available
		# in time (ignoring the possibility that they might transfer
		# to an AG when that flag is cleared)
		_dmvoladm = \
			"dmvoladm -q -l " LS \
			" -c \"count vg=" _vg " and hfree=on\""
		_dmvoladm | getline
		close(_dmvoladm)
		update_time()
		_effective_volumes += $1

		# Add in a share of an AG, if any
		_ag = vg_ag[_vg]
		if (_ag) {
		    # This VG has an AG defined
		    if (vg_ag_max[_vg]) {
			# This VG has an ALLOCATION_MAXIMUM defined
			_dmvoladm = \
				"dmvoladm -q -l " LS " -c \"count vg=" _vg "\""
			_dmvoladm | getline
			close(_dmvoladm)
			update_time()
			_from_ag = _max(vg_ag_max[_vg] - $1, 0)
			_from_ag = _min(_from_ag, ags[_ag])
		    } else {
			# No ALLOCATION_MAXIMUM defined, so entire AG available
			_from_ag = ags[_ag]
		    }
		    _max_volumes += _from_ag

		    # This is the same formula used by the RW and VLM for
		    # discounting the number of tapes in an AG used by
		    # multiple VGs.

		    if (ag_refs[_ag] != 1) {
			_from_ag = \
				int(_from_ag / int(1.5 + sqrt(ag_refs[_ag])));
		    }
		    _effective_volumes += _from_ag
		    _debug(O, "\t\t" _vg " uses AG " _ag \
			    ": eff_volumes=" _effective_volumes \
			    ", _from_ag=" _from_ag)
		} else {
		    # No AG
		    _debug(O, "\t\t" _vg \
			    " doesnt use an AG - eff_volumes=" \
			    _effective_volumes)
		}

		# Must have at least one writable tape to proceed
		if (!_max_volumes) {
		    _log(I, _vg " has no writable volumes - bypassed")
		    continue
		}

		# Scale by MIN_VOLUMES
		_score = _effective_volumes / _max(0.5, vg_minimum[_vg])

		# Save in linked list, sorted by score.  Can"t just pick the
		# single best VG, as it may not be able to have any tapes
		# merged, so we"d need to know the 2nd best one, and maybe the
		# 3rd, and...
		vg_score_val[_vg] = _score
		if (!vg_score_list) {		# first VG
		    vg_score_list = _vg
		} else {
		    _p = ""
		    _n = vg_score_list
		    while (_n && vg_score_val[_n] < _score) {
			_p = _n
			_n = vg_score_nxt[_n]
		    }
		    if (!_p) {			# insert at head of list
			vg_score_list = _vg
		    } else {
			vg_score_nxt[_p] = _vg
		    }
		    vg_score_nxt[_vg] = _n
		}

#_n = vg_score_list
#while (_n) {
#    _debug(O, "\t\t\t\t  score_list: " _n "\t" vg_score_val[_n])
#    _n = vg_score_nxt[_n]
#}

	    }
	    _debug(V, "\t\t-- Leaving _find_best_vg() - returning " vg_score_list)
	    return vg_score_list
	}

	function _sparse_tapes(_vg, _drives_idle,
						_x, _data, _vols,
						_dmvoladm_l, _dmvoladm_s) {
	    _debug(V, "\t\t-- Entering _sparse_tapes(" _vg ", " \
		    _drives_idle ")")
	    _dmvoladm_s = "dmvoladm -q -l " LS
	    _dmvoladm_l = _dmvoladm_s \
		    " -c \"list vg=" _vg " and hfull=on and hoa=off and" \
		    " hlock=off and hsparse=off and dl>0 and threshold<"
	    if (vg_score_val[_vg] < 1) {
		_dmvoladm_l = _dmvoladm_l _min(2 * THRESHOLD, 99)
		# Defer sending a log message about this adjustment to the
		# threshhold until we know if it makes any diffence
	    } else {
		_dmvoladm_l = _dmvoladm_l THRESHOLD
	    }
	    _dmvoladm_l = _dmvoladm_l \
		    " recordorder data format dl\" | sort -n +1"
	    _debug(V, "_dmvoladm_l = \"" _dmvoladm_l "\"")

	    # Skip 3-line heading
	    _dmvoladm_l | getline; _dmvoladm_l | getline; _dmvoladm_l | getline;
	    _data = _vols = 0
	    for (; _drives_idle > 0; _drives_idle--) {
		_x = _dmvoladm_l | getline
		update_time()
		if (_x < 0) {
		    _log(E, "Unable to get tape details from dmvoladm")
		}
		if (_x > 0) {
		    _vols++
		    if (_vols == 1 && vg_score_val[_vg] < 1) {
			_log(O, _vg " has " \
				vg_score_val[_vg] * _max(0.5, vg_minimum[_vg])\
				" effective tapes, below the minimum of " \
				_max(0.5, vg_minimum[_vg]) \
				" - increasing THRESHOLD to " \
				_min(2 * THRESHOLD, 99))
		    }
		    if (VOLUME_LIMIT && _vols > VOLUME_LIMIT) {
			_log(O, "VOLUME_LIMIT of " VOLUME_LIMIT " reached")
			break
		    }
		    if (DEBUG > 1) {
			_log(I, "Would have set HSPARSE on tape " $1)
		    } else {
			_log(I, "Setting HSPARSE on tape " $1)
			print "update " $1 " to hsparse on" | _dmvoladm_s
		    }
		    update_time()
		    starting_up = 0
		    _data += $2
		    if (DATA_LIMIT_EXP && _data * 1000000 > DATA_LIMIT_EXP) {
			_log(O, "DATA_LIMIT of " DATA_LIMIT " reached")
			break
		    }
		    continue
		}
		break
	    }
	    close(_dmvoladm_l)
	    close(_dmvoladm_s)
	    _x = (_vols != 1 ? "s" : "")
	    _log((_vols ? O : V), "Merging " _vols " volume" _x " in VG " _vg)
	    _debug(V, "\t\t-- Leaving _sparse_tapes(" _vg ", " _drives_idle \
		    ") - returning " _vols)
	    return _vols
	}

	BEGIN {
	    stderr = "cat >&2"
	    E = "E"; msg_level[E] = -1
	    O = "O"; msg_level[O] = 0
	    I = "I"; msg_level[I] = 1
	    V = "V"; msg_level[V] = 2
		     msg_level[1] = 3
		     msg_level[2] = 4
		     msg_level[3] = 5
		     msg_level[4] = 6
	    date = "date \"+%T:%3N\""
	    update_time()

	    skip_hdr_processing = 0
	    skip_end_processing = 0

	    # avoid running near the change of minute, as interlock
	    # conflicts are more likely then.
	    s = now
	    sub(/.*:/, "", s)
	    s += 0
	    if (s > 55) {
		system("sleep 10")
		update_time()
	    } else if (s < 5) {
		system("sleep 5")
		update_time()
	    }
	}

	# Get names of VGs

	FILENAME == "-" {
	    if (!NF) {
		_log(E, "no VGs found for DG " DG " - aborting")
		_break_after_exit()
	    }
	    for (x = 1; x <= NF; x++) {
		vgs[$x] = 1
	    }
	    next
	}

#DG_Stats        Version 1.0

	! skip_hdr_processing {
	    if (DEBUG > 0) {
		_log(E, "Running with DEBUG messages")
	    }
	    if (DEBUG > 1) {
		_log(E, "No DB changes being made")
	    }
	    if ($1 != "DG_Stats" || $2 != "Version") {
		_log(E, "awk input is not a DG .txt file - aborting")
		_break_after_exit()
	    }
	    if ($3 + 0 < RW_VERSION + 0) {
		_log(E, "requires RW .txt format " RW_VERSION \
			" or later, not " $3 " - aborting")
		_break_after_exit()
	    }
	    if (int($3) != int(RW_VERSION)) {
		_log(E, "not validated for RW .txt format " \
			int($3) ".x - aborting")
		_break_after_exit()
	    }
	    skip_hdr_processing = 1
	    next
	}

#LibraryServer   ls              2002/12/18 12:19:46     grunt

	$1 == "LibraryServer" {
	    if ($2 != LS) {
		_log(E, "Input is from LS \"" $2 "\", not \"" LS \
			"\" as expected - aborting")
		_break_after_exit()
	    }
	    next
	}

##DG_fields                              assign  down    reserve idle    total
#DriveGroup      dg1       09:35:34        0       0       0       3       3

	$1 == "DriveGroup" && $2 == DG {
	    drives_idle_by_100 = int(_min( \
		    $7 * 100, \
		    prev_drives_idle_by_100 * (1 - LOAD_FN_PERCENT / 100) + \
			$7 * LOAD_FN_PERCENT))
	    drives_idle = int(drives_idle_by_100 / 100 + 0.1)
	    _log(I, DG " drives idle: prv=" prev_drives_idle_by_100 / 100 \
		      " now=" $7 " avg=" drives_idle_by_100 / 100 \
		      " use=" drives_idle)
	    if (!drives_idle) {
		_log(V, "Insufficient idle " DG " drives; waiting " \
			CYCLE_TIME "m")
	        last_update = prev_last_update
		_continue_after_exit()
	    }
	    last_update = $3
	    if (last_update == prev_last_update && !starting_up) {
		_log(O, "RW files unchanged since " prev_last_update \
			"; waiting " CYCLE_TIME "m")
		_continue_after_exit()
	    }
	    next
	}

##VG_fields              allowed assign request empty in-ag partial sparse min
#VolumeGroup  proda 09:35:34  3     0     0       1    N/A    1       0     0
#VolumeGroup  vg1a  09:35:34  3     0     0       0      2    1       0     0

	$1 == "VolumeGroup" {
	    v = $2
	    if (! v in vgs) {	# this VG belongs to a different DG
		next
	    }
	    vg_allowed[v] = $4
	    vg_assigned[v] = $5
	    vg_empty[v] = $7
#	    vg_in_ag[v] = $8
	    vg_partial[v] = $9
	    vg_sparse[v] = $10
	    vg_minimum[v] = $11
	    vg_ag[v] = ""
	    next
	}

#AllocationGroup ag              2

	$1 == "AllocationGroup" {
	    ags[$2] = $3
	    ag_refs[$2] = 0
	    next
	}

	END {
	    if (!skip_end_processing) {
# The plan:
#
# 1. Find the number of drives available, subtracting one if the config
#    forces socket merging - call it "N".  (Ignoring the possibility of
#    insufficient cache requiring a socket merge of large chunks.)
# 2. Associate VGs with their AG, if any
# 3. Find the one VG with the greatest need, based on the number of empty,
#    hsparse, hfree and partial tapes available to it as well as its
#    MIN_VOLUMES.  This may require checking an associated AG.  The result
#    is scaled by MIN_VOLUMES.
# 4. Choose the N emptiest tapes, and set their HSPARSE bits

# 1.
		CACHE_SPACE = _dmconfig(LS " CACHE_SPACE")
		if (!CACHE_SPACE) {
		    _debug(O, "LS " LS " forces socket merges")
		    drives_idle--
		}
		_debug(O, "drives_idle = " drives_idle)
		if (drives_idle > 0) {

# 2.
		    for (v in vgs) {
			vg_ag[v] = _dmconfig(v " ALLOCATION_GROUP")
			if (vg_ag[v]) {
			    ag_refs[vg_ag[v]]++
			}
			vg_ag_max[v] = _dmconfig(v " ALLOCATION_MAXIMUM")
		    }

# 3.
		    if (!(v = _find_best_vg())) {
			_log(O, "No suitable VG found")
			_break_after_exit()
		    } else {
			_debug(O, "\tBest VG found is " v "\t" vg_score_val[v])
			while (vg_score_nxt[v]) {
			    v = vg_score_nxt[v]
			    _debug(I, "\t\t beating " v "\t" vg_score_val[v])
			}
		    }

# 4.
		    vg = vg_score_list
		    while (!_sparse_tapes(vg, _min(drives_idle, \
				_max(0, vg_allowed[vg] - vg_assigned[vg])))) {
			vg = vg_score_nxt[vg]
			if (!vg) {
			    _break_after_exit()
			}
		    }
		}
		_debug(V, "\t\t-- Will iterate again after a sleep")
	    }

	    # Save some info as environment variables in the shell, for use
	    # in the next iteration
	    _setenv("last_update", last_update)
	    _setenv("starting_up", starting_up)
	    _setenv("drives_idle_by_100", drives_idle_by_100)
	}

    ' DEBUG=$DEBUG PID=$$ MYSELF=$MYSELF RW_VERSION=$RW_VERSION \
    LS=$LS DG=$DG TASKGROUP=$TASKGROUP hostname=$(hostname) \
    THRESHOLD=${THRESHOLD:-99} VOLUME_LIMIT=$VOLUME_LIMIT \
    DATA_LIMIT=$DATA_LIMIT DATA_LIMIT_EXP=$DATA_LIMIT_EXP \
    MESSAGE_LEVEL=$MESSAGE_LEVEL \
    CYCLE_TIME=$CYCLE_TIME LOAD_FN_PERCENT=$LOAD_FN_PERCENT \
    prev_last_update=$last_update \
    starting_up=$starting_up \
    prev_drives_idle_by_100=$drives_idle_by_100 \
    - $RW_DIR/$DG.txt >| $TMP/$MYSELF.dot
    if (( $? != 0 )); then
	break
    fi
    . $TMP/$MYSELF.dot
    sleep $(( $CYCLE_TIME * 60 ))
done

# And go

rm -f $TMP/*
_log O "Merge Manager $TASKGROUP (DG $DG) ending"
if [[ -f $POSTDMALERT ]]; then
    $POSTDMALERT -p 3 -c RUNMERGEMGR000 -s task -k \
    "cmd:$THISCMD,taskgroup:$TASKGROUP" \
    "$MYSELF completed successfully." \
    >/dev/null 2>&1
fi
#endscript run_merge_mgr