recursive-delete.js 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251
  1. "use strict";
  2. Object.defineProperty(exports, "__esModule", { value: true });
  3. exports.RecursiveDelete = exports.RECURSIVE_DELETE_MIN_PENDING_OPS = exports.RECURSIVE_DELETE_MAX_PENDING_OPS = exports.REFERENCE_NAME_MIN_ID = void 0;
  4. const assert = require("assert");
  5. const _1 = require(".");
  6. const util_1 = require("./util");
  7. const query_options_1 = require("./reference/query-options");
  8. /*!
  9. * Datastore allowed numeric IDs where Firestore only allows strings. Numeric
  10. * IDs are exposed to Firestore as __idNUM__, so this is the lowest possible
  11. * negative numeric value expressed in that format.
  12. *
  13. * This constant is used to specify startAt/endAt values when querying for all
  14. * descendants in a single collection.
  15. */
  16. exports.REFERENCE_NAME_MIN_ID = '__id-9223372036854775808__';
  17. /*!
  18. * The query limit used for recursive deletes when fetching all descendants of
  19. * the specified reference to delete. This is done to prevent the query stream
  20. * from streaming documents faster than Firestore can delete.
  21. */
  22. // Visible for testing.
  23. exports.RECURSIVE_DELETE_MAX_PENDING_OPS = 5000;
  24. /*!
  25. * The number of pending BulkWriter operations at which RecursiveDelete
  26. * starts the next limit query to fetch descendants. By starting the query
  27. * while there are pending operations, Firestore can improve BulkWriter
  28. * throughput. This helps prevent BulkWriter from idling while Firestore
  29. * fetches the next query.
  30. */
  31. exports.RECURSIVE_DELETE_MIN_PENDING_OPS = 1000;
  32. /**
  33. * Class used to store state required for running a recursive delete operation.
  34. * Each recursive delete call should use a new instance of the class.
  35. * @private
  36. * @internal
  37. */
  38. class RecursiveDelete {
  39. /**
  40. *
  41. * @param firestore The Firestore instance to use.
  42. * @param writer The BulkWriter instance to use for delete operations.
  43. * @param ref The document or collection reference to recursively delete.
  44. * @param maxLimit The query limit to use when fetching descendants
  45. * @param minLimit The number of pending BulkWriter operations at which
  46. * RecursiveDelete starts the next limit query to fetch descendants.
  47. */
  48. constructor(firestore, writer, ref, maxLimit, minLimit) {
  49. this.firestore = firestore;
  50. this.writer = writer;
  51. this.ref = ref;
  52. this.maxLimit = maxLimit;
  53. this.minLimit = minLimit;
  54. /**
  55. * The number of deletes that failed with a permanent error.
  56. * @private
  57. * @internal
  58. */
  59. this.errorCount = 0;
  60. /**
  61. * Whether there are still documents to delete that still need to be fetched.
  62. * @private
  63. * @internal
  64. */
  65. this.documentsPending = true;
  66. /**
  67. * Whether run() has been called.
  68. * @private
  69. * @internal
  70. */
  71. this.started = false;
  72. /**
  73. * A deferred promise that resolves when the recursive delete operation
  74. * is completed.
  75. * @private
  76. * @internal
  77. */
  78. this.completionDeferred = new util_1.Deferred();
  79. /**
  80. * Whether a query stream is currently in progress. Only one stream can be
  81. * run at a time.
  82. * @private
  83. * @internal
  84. */
  85. this.streamInProgress = false;
  86. /**
  87. * The number of pending BulkWriter operations. Used to determine when the
  88. * next query can be run.
  89. * @private
  90. * @internal
  91. */
  92. this.pendingOpsCount = 0;
  93. this.errorStack = '';
  94. this.maxPendingOps = maxLimit;
  95. this.minPendingOps = minLimit;
  96. }
  97. /**
  98. * Recursively deletes the reference provided in the class constructor.
  99. * Returns a promise that resolves when all descendants have been deleted, or
  100. * if an error occurs.
  101. */
  102. run() {
  103. assert(!this.started, 'RecursiveDelete.run() should only be called once.');
  104. // Capture the error stack to preserve stack tracing across async calls.
  105. this.errorStack = Error().stack;
  106. this.writer._verifyNotClosed();
  107. this.setupStream();
  108. return this.completionDeferred.promise;
  109. }
  110. /**
  111. * Creates a query stream and attaches event handlers to it.
  112. * @private
  113. * @internal
  114. */
  115. setupStream() {
  116. const stream = this.getAllDescendants(this.ref instanceof _1.CollectionReference
  117. ? this.ref
  118. : this.ref);
  119. this.streamInProgress = true;
  120. let streamedDocsCount = 0;
  121. stream
  122. .on('error', err => {
  123. err.code = 14 /* StatusCode.UNAVAILABLE */;
  124. err.stack = 'Failed to fetch children documents: ' + err.stack;
  125. this.lastError = err;
  126. this.onQueryEnd();
  127. })
  128. .on('data', (snap) => {
  129. streamedDocsCount++;
  130. this.lastDocumentSnap = snap;
  131. this.deleteRef(snap.ref);
  132. })
  133. .on('end', () => {
  134. this.streamInProgress = false;
  135. // If there are fewer than the number of documents specified in the
  136. // limit() field, we know that the query is complete.
  137. if (streamedDocsCount < this.minPendingOps) {
  138. this.onQueryEnd();
  139. }
  140. else if (this.pendingOpsCount === 0) {
  141. this.setupStream();
  142. }
  143. });
  144. }
  145. /**
  146. * Retrieves all descendant documents nested under the provided reference.
  147. * @param ref The reference to fetch all descendants for.
  148. * @private
  149. * @internal
  150. * @return {Stream<QueryDocumentSnapshot>} Stream of descendant documents.
  151. */
  152. getAllDescendants(ref) {
  153. // The parent is the closest ancestor document to the location we're
  154. // deleting. If we are deleting a document, the parent is the path of that
  155. // document. If we are deleting a collection, the parent is the path of the
  156. // document containing that collection (or the database root, if it is a
  157. // root collection).
  158. let parentPath = ref._resourcePath;
  159. if (ref instanceof _1.CollectionReference) {
  160. parentPath = parentPath.popLast();
  161. }
  162. const collectionId = ref instanceof _1.CollectionReference
  163. ? ref.id
  164. : ref.parent.id;
  165. let query = new _1.Query(this.firestore, query_options_1.QueryOptions.forKindlessAllDescendants(parentPath, collectionId,
  166. /* requireConsistency= */ false));
  167. // Query for names only to fetch empty snapshots.
  168. query = query.select(_1.FieldPath.documentId()).limit(this.maxPendingOps);
  169. if (ref instanceof _1.CollectionReference) {
  170. // To find all descendants of a collection reference, we need to use a
  171. // composite filter that captures all documents that start with the
  172. // collection prefix. The MIN_KEY constant represents the minimum key in
  173. // this collection, and a null byte + the MIN_KEY represents the minimum
  174. // key is the next possible collection.
  175. const nullChar = String.fromCharCode(0);
  176. const startAt = collectionId + '/' + exports.REFERENCE_NAME_MIN_ID;
  177. const endAt = collectionId + nullChar + '/' + exports.REFERENCE_NAME_MIN_ID;
  178. query = query
  179. .where(_1.FieldPath.documentId(), '>=', startAt)
  180. .where(_1.FieldPath.documentId(), '<', endAt);
  181. }
  182. if (this.lastDocumentSnap) {
  183. query = query.startAfter(this.lastDocumentSnap);
  184. }
  185. return query.stream();
  186. }
  187. /**
  188. * Called when all descendants of the provided reference have been streamed
  189. * or if a permanent error occurs during the stream. Deletes the developer
  190. * provided reference and wraps any errors that occurred.
  191. * @private
  192. * @internal
  193. */
  194. onQueryEnd() {
  195. this.documentsPending = false;
  196. if (this.ref instanceof _1.DocumentReference) {
  197. this.writer.delete(this.ref).catch(err => this.incrementErrorCount(err));
  198. }
  199. this.writer.flush().then(async () => {
  200. var _a;
  201. if (this.lastError === undefined) {
  202. this.completionDeferred.resolve();
  203. }
  204. else {
  205. let error = new (require('google-gax/build/src/fallback').GoogleError)(`${this.errorCount} ` +
  206. `${this.errorCount !== 1 ? 'deletes' : 'delete'} ` +
  207. 'failed. The last delete failed with: ');
  208. if (this.lastError.code !== undefined) {
  209. error.code = this.lastError.code;
  210. }
  211. error = (0, util_1.wrapError)(error, this.errorStack);
  212. // Wrap the BulkWriter error last to provide the full stack trace.
  213. this.completionDeferred.reject(this.lastError.stack
  214. ? (0, util_1.wrapError)(error, (_a = this.lastError.stack) !== null && _a !== void 0 ? _a : '')
  215. : error);
  216. }
  217. });
  218. }
  219. /**
  220. * Deletes the provided reference and starts the next stream if conditions
  221. * are met.
  222. * @private
  223. * @internal
  224. */
  225. deleteRef(docRef) {
  226. this.pendingOpsCount++;
  227. this.writer
  228. .delete(docRef)
  229. .catch(err => {
  230. this.incrementErrorCount(err);
  231. })
  232. .then(() => {
  233. this.pendingOpsCount--;
  234. // We wait until the previous stream has ended in order to sure the
  235. // startAfter document is correct. Starting the next stream while
  236. // there are pending operations allows Firestore to maximize
  237. // BulkWriter throughput.
  238. if (this.documentsPending &&
  239. !this.streamInProgress &&
  240. this.pendingOpsCount < this.minPendingOps) {
  241. this.setupStream();
  242. }
  243. });
  244. }
  245. incrementErrorCount(err) {
  246. this.errorCount++;
  247. this.lastError = err;
  248. }
  249. }
  250. exports.RecursiveDelete = RecursiveDelete;
  251. //# sourceMappingURL=recursive-delete.js.map