!=========================================================================
! Copyright (C) VMware, Inc. 1986-2011.  All Rights Reserved.
!
! $Id: IndexManager.gs,v 1.30 2008-01-09 22:50:07 stever Exp $
!
! Superclass Hierarchy:
!  IndexManager, Object.
!
!=========================================================================

! class created in idxclasses.topaz

removeallmethods IndexManager
removeallclassmethods IndexManager

category: 'For Documentation Installation only'
classmethod: IndexManager
installDocumentation

| doc txt |
doc := GsClassDocumentation newForClass: self.

txt := (GsDocText new) details:
'IndexManager is provides protocol for managing and maintaining indexed collections.
 The single instance of IndexManager should be accessed via the class message #current.
 In normal use, you will use IndexManager to turn autoCommit for index maintenance
 operations off and on and to define the dirtyObjectCommitThreshold and 
 percentTempObjSpaceCommitThreshold, which are control parameters for autoCommit.

 IndexManager current tracks all instances of UnorderedCollection that have had indexes
 created for them. You may use IndexManager current to #removeAllIndexes, which removes
 indexes from all indexed collections in the system or you may use IndexManager 
 current to #removeAllIncompleteIndexes, which removes partial indexes from collections
 when index creation failed without completing.

 Automatic garbage collection of unreferenced UnorderedCollections is _not_ preformed, so
 one must explicitly remove all indexes from an UnorderedCollection that is no longer used.

 IndexManager may be subclassed.'.
doc documentClassWith: txt.

self description: doc.
%
! ------------------- Class methods for IndexManager
category: 'Accessing'
classmethod: IndexManager
current

  Current == nil ifTrue: [ Current := IndexManager basicNew initialize ].
  ^ Current
%

category: 'Accessing'
classmethod: IndexManager
autoCommit

^self current autoCommit
%

category: 'Index Maintenance'
classmethod: IndexManager
removeAllIndexes

^ self current removeAllIndexes.
%

category: 'Index Maintenance'
classmethod: IndexManager
removeAllIncompleteIndexes

^ self current removeAllIncompleteIndexes.
%

category: 'Index Maintenance'
classmethod: IndexManager
cleanupDependencyLists

  "Iterate through all dependency lists, cleaning up entries that have removed indexes."

  self current cleanupDependencyLists
%

category: 'Updating'
classmethod: IndexManager
dirtyObjectCommitThreshold: anInteger

self current dirtyObjectCommitThreshold: anInteger
%

category: 'Updating'
classmethod: IndexManager
percentTempObjSpaceCommitThreshold: anInteger

self current percentTempObjSpaceCommitThreshold: anInteger
%

category: 'Updating'
classmethod: IndexManager
autoCommit: aBool

self current autoCommit: aBool
%

! ------------------- Instance methods for IndexManager

! deleted IndexManager>>addIndex: renamed to IndexManager>>_addIndex: v2.1

category: 'Accessing'
method: IndexManager
_addIndex: anIndex
  "  Add the given index into the collection of all indexes."

  anIndex changeToSegment: GsIndexingSegment.
  self getAllIndexes add: anIndex
%
category: 'Index Maintenance'
method: IndexManager
cleanupDependencyLists

  "Iterate through all dependency lists, cleaning up entries that have removed indexes."
%
category: 'Index Updating'
method: IndexManager
createRcEqualityIndexFor: anNSC on: aPathString withLastElementClass: aClass

  "Replaces UnorderedCollection>>_createEqualityIndexOn:withLastElementClass:commitInterval:, modulo the error handler"


  ( ( aClass canUnderstand: #<= ) _or:
  [ aClass == Boolean _or: [ aClass == UndefinedObject ] ] )
    ifFalse: [
      ^ #[ #rtErrBagClassDoesNotSupportRangeOperators, #[ aClass ]]
    ].
^ self _createIndex: self _equalityIndex 
        for: anNSC 
        on: aPathString 
        withLastElementClass: aClass 
        equalityIndexClass:  RcRangeEqualityIndex
%
category: 'Index Updating'
method: IndexManager
createEqualityIndexFor: anNSC on: aPathString withLastElementClass: aClass

  "Replaces UnorderedCollection>>_createEqualityIndexOn:withLastElementClass:commitInterval:, modulo the error handler"


  ( ( aClass canUnderstand: #<= ) _or:
  [ aClass == Boolean _or: [ aClass == UndefinedObject ] ] )
    ifFalse: [
      ^ #[ #rtErrBagClassDoesNotSupportRangeOperators, #[ aClass ]]
    ].
  ^self _createIndex: self _equalityIndex for: anNSC on: aPathString withLastElementClass: aClass
%
category: 'Index Updating'
method: IndexManager
createIdentityIndexFor: anNSC on: aPathString

  "Replaces UnorderedCollection>>_createIdentityIndexOn:commitInterval:"

  ^self _createIndex: self _identityIndex for: anNSC on: aPathString withLastElementClass: nil
%
category: 'Accessing'
method: IndexManager
getAllIndexes

  "Return a collection of all indexes in the system."

  allIndexes == nil 
    ifTrue: [
      allIndexes := RcIdentityBag new.
      allIndexes changeToSegment: GsIndexingSegment.
    ].
  ^allIndexes
%
category: 'Accessing'
method: IndexManager
getAllNSCRoots

  "Return a collection of all nsc's with indexes in the system."

  | set |
  set := IdentitySet new.
  self getAllIndexes do: [:index |
    set add: index nscRoot
  ].
  ^set
%
category: 'Index Maintenance'
method: IndexManager
removeAllIndexes

  | result |
  self getAllNSCRoots do: [:nsc | 
     result := self removeAllIndexesOn: nsc.
    (nsc == result)
      ifFalse: [ self _error: #rtErrRemoveAllIndexesFailed args: #[ nsc,  result ] ].
  ].
%
category: 'Index Maintenance'
method: IndexManager
removeAllIncompleteIndexes

  self getAllNSCRoots do: [:nsc | 
    self removeAllIncompleteIndexesOn: nsc
  ].
%
category: 'Index Updating'
method: IndexManager
removeAllIndexesOn: anNSC

  | result |
result := self removeAllCompleteIndexesOn: anNSC.
result == anNSC
  ifTrue: [
    self removeAllIncompleteIndexesOn: anNSC.
  ].
^result
%
category: 'Index Updating'
method: IndexManager
removeAllCompleteIndexesOn: anNSC
  "Remove all complete indexes for anNSC. If all of the anNSC's indexes can be removed, this method returns the receiver."

  "If an error occurs during index removal, it may not be possible to commit the current transaction later."

  | iList rootIndexes hasNonRootIndex result |
  (iList := anNSC _indexedPaths) == nil
    ifTrue: [
      ^ anNSC
    ].

  hasNonRootIndex := false.
  rootIndexes := Array new.
  " scan each entry to see if a non-root index is present "
  1 to: iList size by: 2 do: [ :i |
    " if it is a root index ... "
    (((iList at: i + 1) == 1) _and: [ (iList at: i) isComplete])
        " add it to a list "
        ifTrue: [ rootIndexes add: (iList at: i) ]
        " otherwise flag that we're really not removing ALL indexes "
        ifFalse: [ hasNonRootIndex := true ]
  ].

  Exception
    category: nil
    number: nil
    do: [ :ex :cat :num :args | | txt |
        " get the text for the raised error "
        txt := cat textForError: num args: args.
        " check for recursive signal "
        num == (ErrorSymbols at: #rtErrPreventingCommit)
            ifTrue: [ " remove this exception and resignal "
                ex resignal: cat number: num args: args
            ]
            ifFalse: [ " append additional message to the end of text "
                txt _error: #rtErrPreventingCommit
            ]
    ].
  IndexManager current executeStartingIndexMaintenance: [
    result := anNSC 
                _removeAllRootIndexes: rootIndexes 
                hasNonRootIndex: hasNonRootIndex.
    result == anNSC
      ifTrue: [
        rootIndexes do: [:each |
          self _removeIndex: each.
        ].
    ].
  ].

  ^ result
%

category: 'Index Updating'
method: IndexManager
removeAllIncompleteIndexesOn: anNSC

  "Remove all incomplete indexes for anNSC. If all of the anNSC's indexes can be removed, this method returns the receiver."

  | iList rootIndexes |
  (iList := anNSC _indexedPaths) == nil
    ifTrue: [
      ^ anNSC
    ].

  rootIndexes := Array new.
  " scan each entry to see if a non-root index is present "
  1 to: iList size by: 2 do: [ :i |
    " if it is an incomplete root index ... "
    (((iList at: i + 1) == 1) _and: [ (iList at: i) isComplete not])
        " add it to a list "
        ifTrue: [ rootIndexes add: (iList at: i) ]
  ].

  IndexManager current executeStartingIndexMaintenance: [
        rootIndexes do: [:indexObj |
           indexObj nscRoot _undoIndexCreation: indexObj pathTerm: indexObj _findFirstUnsharedPathTerm.
          self _removeIndex: indexObj.
    ].
  ].
^rootIndexes size
%

category: 'Index Updating'
method: IndexManager
removeEqualityIndexFor: anNSC on: aPathString

  self _removeIndex: self _equalityIndex for: anNSC on: aPathString
%
category: 'Index Updating'
method: IndexManager
removeIdentityIndexFor: anNSC on: aPathString

  self _removeIndex: self _identityIndex for: anNSC on: aPathString
%

! deleted IndexManager>>removeIndex: renamed to IndexManager>>_removeIndex: v2.1

category: 'Accessing'
method: IndexManager
_removeIndex: anIndex

  "Remove the given index from the collection of all indexes.
   Make sure that IVs are niled in anIndex, since it will
   survive (in rc redo log) until abort or commit."

  self getAllIndexes remove: anIndex ifAbsent: [ nil ].
  anIndex _clear.
%
category: 'Accessing'
method: IndexManager
resetAllIndexes

  allIndexes := nil.
%
category: 'Index Updating - Private'
method: IndexManager
_createIndex: indexType for: anNSC on: aPathString withLastElementClass: aClass

^ self _createIndex: indexType 
        for: anNSC on: aPathString 
        withLastElementClass: aClass 
        equalityIndexClass:  RangeEqualityIndex
%

category: 'Index Updating - Private'
method: IndexManager
_createIndex: indexType for: anNSC on: aPathString withLastElementClass: aClass equalityIndexClass:  aRangeEqualityIndex

  | pathArray index indexes  lastConstraint updateIndexListSegments original | 

  " see if incomplete index for same path "
  anNSC _hasIncompleteIndexes
    ifTrue: [ ^self _error: #rtErrCollectionWithIncompleteIndex].

  Exception
    category: nil
    number: nil
    do: [ :ex :cat :num :args | | txt |
      " get the text for the raised error "
      txt := cat textForError: num args: args.
      " check for recursive signal "
      num == (ErrorSymbols at: #rtErrPreventingCommit)
        ifTrue: [ " remove this exception and resignal "
          self autoCommit
            ifTrue: [ anNSC removeIncompleteIndex ].
          ex resignal: cat number: num args: args
        ]
        ifFalse: [ " append additional message to the end of text "
          ex remove.
          txt _error: #rtErrPreventingCommit
        ]
    ].

  pathArray := aPathString asArrayOfPathTerms.
  " check if an identity index already exists for the path "
  indexes := anNSC  _findIndexesWithPath: pathArray.
  indexes do: [ :idxObj |
    (indexType == self _identityIndex)
      ifTrue: [
        idxObj isIdentityIndex ifTrue: [ ^ nil ]
      ]
      ifFalse: [
        idxObj isRangeEqualityIndex ifTrue: [ ^ nil ]
      ].
  ].

  "No index already exists for collection, lets create one"
  (indexType == self _identityIndex)
    ifTrue: [
      lastConstraint := anNSC getLastElementConstraintOnPath: aPathString.
      ( lastConstraint ~~ nil _and: [ anNSC _isRangeable: lastConstraint ] )
        ifTrue: [
          1 to: indexes size do: [ :i |
            (indexes at: i) isRangeEqualityIndex ifTrue: [ ^ nil ]
          ].
          index := aRangeEqualityIndex newWithLastElementClass: lastConstraint
        ]
        ifFalse: [ index := IdentityIndex new ].
    ]
    ifFalse: [
        index := aRangeEqualityIndex newWithLastElementClass: aClass.
    ].

  " don't need index dictionary if path length is one "
  ( pathArray size == 1 _and: [ index isRangeEqualityIndex ] )
    ifFalse: [
      | indexDict |
      anNSC _calculateIndexDictionarySize: pathArray size.
      indexDict := anNSC _getIndexDictionary.
      index indexDictionary: indexDict.
      "afford an opportunity to resize dictionary before adding new elements"
      indexDict rebuildTable: anNSC _getIndexDictionaryCreationSize for: index.
    ].
  index nscRoot: anNSC.

  "Not certain that we need to worry about updateIndexListSegments"
  (indexType == self _identityIndex)
    ifTrue: [
      updateIndexListSegments := anNSC _indexedPaths == nil.
    ]
    ifFalse: [
      updateIndexListSegments :=
        anNSC _indexedPaths == nil _or: [ anNSC _indexedPaths _hasExplicitIndex not ].
    ].

  anNSC _getIndexList buildPathTermsFor: index with: pathArray.
  anNSC _putInWriteSet.

  self _addIndex: index.

  original := index preIndexCreation.

  " optimize creation of indexes on elements of the NSC itself "
  index isIndexOnRootNsc
    ifTrue: [
      index  addDirectMappingsFor: anNSC _asIdentityBag
        indexList: anNSC _indexedPaths
        updateIndexListSegments: updateIndexListSegments
    ]
    ifFalse: [
      self _addMappingsFor: anNSC
        on: index
        updateIndexListSegments: updateIndexListSegments
    ].

  index postIndexCreation: original.

  ^ nil
%

category: 'Private'
method: IndexManager
_doCommit
| systm |
    systm := System .
    systm commitTransaction ifFalse: 
        [self _errorCouldNotCommitDuringIndexCreation].
    systm transactionMode == #manualBegin ifTrue:
        [systm beginTransaction].
%
category: 'Private'
method: IndexManager
_equalityIndex

  ^#equality
%
category: 'Private'
method: IndexManager
_identityIndex

  ^#identity
%
category: 'Index Updating - Private'
method: IndexManager
_removeIndex: indexType for: anNSC on: aPathString

  | iList indexes index pathArray |
  anNSC _checkIndexPathExpression: aPathString.

  iList := anNSC _indexedPaths.
  iList == nil
    ifTrue: [ self _error: #rtErrNoIndexForPath args: #[ aPathString ] ].

  pathArray := aPathString asArrayOfPathTerms.
  " check if an index exists for the path string "
  indexes := anNSC _findIndexesWithPath: pathArray.

  indexes isEmpty
    ifFalse: [
      " make sure it is an identity index or equality index"
      indexes do: [ :iObj |
        (indexType == self _identityIndex)
          ifTrue: [
            ( iObj isIdentityIndex _or:
              [ anNSC _isRangeable: iObj lastElementClass ] )
              ifTrue: [ index := iObj ]
          ]
          ifFalse: [
            iObj isRangeEqualityIndex
              ifTrue: [ index := iObj ]
          ]
      ].
    ].
  " no index was found with the given path "
  index == nil
    ifTrue: [ ^ self _error: #rtErrNoIndexForPath args: #[ aPathString ]  ].

  Exception
    category: nil
    number: nil
    do: [ :ex :cat :num :args | | txt |
    " get the text for the raised error "
    txt := cat textForError: num args: args.
    " check for recursive signal "
    num == (ErrorSymbols at: #rtErrPreventingCommit)
      ifTrue: [ " remove this exception and resignal "
        ex resignal: cat number: num args: args
      ]
      ifFalse: [ " append additional message to the end of text "
        txt _error: #rtErrPreventingCommit
      ]
    ]. 

  IndexManager current executeStartingIndexMaintenance: [
    index preIndexRemoval.
    anNSC _removeIndex: index.
    self _removeIndex: index.
  ].
%

category: 'Testing'
method: IndexManager
shouldCommit
"Answer true if a commit should be performed during index maintenance.
   Note - that this method should only be executed from within a
   executeStartingIndexMaintenance: block, since _autoCommitPolicy
   may not have been initialized correctly otherwise."

  | policy |
self autoCommit
  ifFalse: [ ^ false ].
policy := self _autoCommitPolicy.
(policy == nil _or: [ "bug34941"policy == true]) ifTrue: [ ^ false ].
^ policy shouldCommit
%

category: 'Testing'
method: IndexManager
_autoCommitPolicy

^(System _sessionStateAt: 11)
%

category: 'Testing'
method: IndexManager
_clearAutoCommitPolicy

System _sessionStateAt: 11 put: nil
%

category: 'Testing'
method: IndexManager
_initializeAutoCommitPolicy

System _sessionStateAt: 11 put: (IndexManagerAutoCommitPolicy on: self)
%

category: 'Initializing'
method: IndexManager
initialize

  "When creating initial instance, make sure all IVs are set, since we don't
   want to dirty the object during normal operation."

  self autoCommit: false.       
  self getAllIndexes.
  self useKeyEncryption: true.
  "20000 objects is a good size for a commit"
  self dirtyObjectCommitThreshold: 20000.
  "With 75% threshold, we can run along at just under 100% TempObjSpacePercentUsed"
  self percentTempObjSpaceCommitThreshold: 75.

  self changeToSegment: GsIndexingSegment.
%

category: 'Accessing'
method: IndexManager
autoCommit

^autoCommit
%

category: 'Accessing'
method: IndexManager
useKeyEncryption
  
  ^useKeyEncryption
%

category: 'Updating'
method: IndexManager
useKeyEncryption: aBool
  
  useKeyEncryption := aBool

%

category: 'Updating'
method: IndexManager
autoCommit: aBool

autoCommit := aBool
%

category: 'Accessing'
method: IndexManager
dirtyObjectCommitThreshold

^ dirtyObjectCommitThreshold
%

category: 'Accessing'
method: IndexManager
rcBtreeLeafNodeClass

  ^ RcBtreeLeafNode
%

category: 'Accessing'
method: IndexManager
rcBtreeBasicLeafNodeClass

  self useKeyEncryption
    ifTrue: [ ^ RcBtreeBasicLeafNode ].
  ^ RcBtreeLeafNode
%

category: 'Accessing'
method: IndexManager
btreeBasicLeafNodeClass

  self useKeyEncryption
    ifTrue: [ ^ BtreeBasicLeafNode ].
  ^ BtreeLeafNode
%

category: 'Accessing'
method: IndexManager
btreeLeafNodeClass

  ^ BtreeLeafNode
%

category: 'Accessing'
method: IndexManager
sortNodeClass

  self useKeyEncryption
    ifTrue: [ ^ BasicSortNode ].
  ^ SortNode
%

category: 'Updating'
method: IndexManager
dirtyObjectCommitThreshold: anInteger

dirtyObjectCommitThreshold := anInteger
%

category: 'Accessing'
method: IndexManager
percentTempObjSpaceCommitThreshold

^ percentTempObjSpaceCommitThreshold
%

category: 'Updating'
method: IndexManager
percentTempObjSpaceCommitThreshold: anInteger

percentTempObjSpaceCommitThreshold := anInteger
%

category: 'Updating'
method: IndexManager
changeToSegment: aSegment

"Assigns the receiver and the allIndexes collection to aSegment."

<primitive: 901>

aSegment == GsIndexingSegment
  ifFalse: [ self _error: #segmentNotSharedDepListSegment ].

self assignToSegment: aSegment.
self getAllIndexes changeToSegment: aSegment.

System _disableProtectedMode.
%

category: 'Index Maintenance'
method: IndexManager
commitIndexMaintenance: indexObj at: progressCount

self shouldCommit
  ifTrue: [
    indexObj progress: progressCount.
    self _doCommit.
  ].
%

category: 'Index Maintenance'
method: IndexManager
executeStartingIndexMaintenance: aBlock
"Behavior of index maintenance with respect to transaction state:

When in manual mode:
    When IndexManager current autoCommit is true:
        When originally outside of a transaction:
            Begin a transaction.
            Commit the final transaction of index creation.
            End outside of a transaction.
        When originally inside of a transaction:
            Commit the final transaction of index creation.
            End outside of a transaction.
When in auto mode:
    When IndexManager current autoCommit is true:
        Commit the final transaction of index creation.
"


  | inAutoCommit autoCommitPolicy |

inAutoCommit := self autoCommit.
inAutoCommit
  ifTrue: [ | systm |
    systm := System .
    systm transactionMode == #manualBegin 
      ifTrue:
        [systm inTransaction 
           ifFalse:
             [systm needsCommit
                ifTrue: [ self _error: #rtErrAbortWouldLoseData ] .
              systm beginTransaction]].
    self _initializeAutoCommitPolicy].

autoCommitPolicy := self _autoCommitPolicy.
(autoCommitPolicy == nil _or: [ "bug34941"autoCommitPolicy == true])
  ifTrue: [
    aBlock value.
  ]
  ifFalse: [
    [ autoCommitPolicy evaluate: aBlock for: self ] 
      ensure: [ 
        inAutoCommit ifTrue: [ self _clearAutoCommitPolicy ].
      ].
  ].
inAutoCommit
  ifTrue:
    [System commitTransaction ifFalse: 
       [self _errorCouldNotCommitDuringIndexCreation].
    ].
%

category: 'Updating Indexes - Private'
method: IndexManager
_addMappingsFor: anNSC
on: indexObj
updateIndexListSegments: aBoolean

"Adds the mappings in the index dictionary, dependency lists, and so on for all
 elements in the receiver."

| pathTerm object bag iList |

" since this index may use path terms from other indexes,
find the first one that is not shared by another index (only
need to add mappings from that point onward) "
iList := anNSC _indexedPaths.

pathTerm := indexObj _findFirstUnsharedPathTerm.
pathTerm == nil
  ifTrue: [ " index is subsumed by an existing index "
    ^ anNSC
  ].

bag := anNSC _asIdentityBag.
self autoCommit
  ifTrue: [ anNSC _lockForIndexCreation ].

" see if need to traverse to the first unshared path term "
indexObj firstPathTerm == pathTerm
  ifTrue: [
    " if index list's segments set needs updated ... "
    aBoolean
      ifTrue: [
        1 to: bag size do: [ :i |
          object := bag _at: i.
          pathTerm addMappingsForObject: object logging: false.
          self commitIndexMaintenance: indexObj at: i.
        ]
      ]
      ifFalse: [
        1 to: bag size do: [ :i |
          pathTerm addMappingsForObject: (bag _at: i) logging: false.
          self commitIndexMaintenance: indexObj at: i.
        ]
      ]
  ]
  ifFalse: [
    " if index list's segments set needs updated ... "
    aBoolean
      ifTrue: [
        1 to: bag size do: [ :i |
          object := bag _at: i.
          " traverse up to the point of the unshared path term "
          object := indexObj traverse: object upTo: pathTerm.
          pathTerm addMappingsForObject: object logging: false.
          self commitIndexMaintenance: indexObj at: i.
        ]
      ]
      ifFalse: [
        1 to: bag size do: [ :i |
          " traverse up to the point of the unshared path term "
          object := indexObj traverse: (bag _at: i) upTo: pathTerm.
          pathTerm addMappingsForObject: object logging: false.
          self commitIndexMaintenance: indexObj at: i.
        ]
      ]
  ].
^ anNSC
%

! make sure default values are initialized
run
IndexManager current.
^true
%
