View Javadoc

1   /**
2    *
3    * Copyright (C) 2011 Cloud Conscious, LLC. <info@cloudconscious.com>
4    *
5    * ====================================================================
6    * Licensed under the Apache License, Version 2.0 (the "License");
7    * you may not use this file except in compliance with the License.
8    * You may obtain a copy of the License at
9    *
10   * http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   * ====================================================================
18   */
19  package org.jclouds.compute.internal;
20  
21  import static com.google.common.base.Preconditions.checkNotNull;
22  import static com.google.common.base.Predicates.and;
23  import static com.google.common.base.Predicates.not;
24  import static com.google.common.base.Predicates.notNull;
25  import static com.google.common.collect.Iterables.filter;
26  import static com.google.common.collect.Maps.newLinkedHashMap;
27  import static com.google.common.collect.Sets.filter;
28  import static com.google.common.collect.Sets.newLinkedHashSet;
29  import static com.google.common.util.concurrent.Futures.immediateFuture;
30  import static org.jclouds.compute.predicates.NodePredicates.TERMINATED;
31  import static org.jclouds.compute.predicates.NodePredicates.all;
32  import static org.jclouds.concurrent.FutureIterables.awaitCompletion;
33  import static org.jclouds.concurrent.FutureIterables.transformParallel;
34  
35  import java.io.IOException;
36  import java.util.Map;
37  import java.util.NoSuchElementException;
38  import java.util.Set;
39  import java.util.concurrent.Callable;
40  import java.util.concurrent.ExecutorService;
41  import java.util.concurrent.Future;
42  import java.util.concurrent.TimeUnit;
43  import java.util.concurrent.atomic.AtomicReference;
44  
45  import javax.annotation.Resource;
46  import javax.inject.Inject;
47  import javax.inject.Named;
48  import javax.inject.Provider;
49  import javax.inject.Singleton;
50  
51  import org.jclouds.Constants;
52  import org.jclouds.collect.Memoized;
53  import org.jclouds.compute.ComputeService;
54  import org.jclouds.compute.ComputeServiceContext;
55  import org.jclouds.compute.RunNodesException;
56  import org.jclouds.compute.RunScriptOnNodesException;
57  import org.jclouds.compute.callables.RunScriptOnNode;
58  import org.jclouds.compute.config.CustomizationResponse;
59  import org.jclouds.compute.domain.ComputeMetadata;
60  import org.jclouds.compute.domain.ExecResponse;
61  import org.jclouds.compute.domain.Hardware;
62  import org.jclouds.compute.domain.Image;
63  import org.jclouds.compute.domain.NodeMetadata;
64  import org.jclouds.compute.domain.NodeMetadataBuilder;
65  import org.jclouds.compute.domain.NodeState;
66  import org.jclouds.compute.domain.Template;
67  import org.jclouds.compute.domain.TemplateBuilder;
68  import org.jclouds.compute.options.RunScriptOptions;
69  import org.jclouds.compute.options.TemplateOptions;
70  import org.jclouds.compute.reference.ComputeServiceConstants;
71  import org.jclouds.compute.reference.ComputeServiceConstants.Timeouts;
72  import org.jclouds.compute.strategy.CreateNodesInGroupThenAddToSet;
73  import org.jclouds.compute.strategy.DestroyNodeStrategy;
74  import org.jclouds.compute.strategy.GetNodeMetadataStrategy;
75  import org.jclouds.compute.strategy.InitializeRunScriptOnNodeOrPlaceInBadMap;
76  import org.jclouds.compute.strategy.ListNodesStrategy;
77  import org.jclouds.compute.strategy.RebootNodeStrategy;
78  import org.jclouds.compute.strategy.ResumeNodeStrategy;
79  import org.jclouds.compute.strategy.RunScriptOnNodeAndAddToGoodMapOrPutExceptionIntoBadMap;
80  import org.jclouds.compute.strategy.SuspendNodeStrategy;
81  import org.jclouds.domain.Credentials;
82  import org.jclouds.domain.Location;
83  import org.jclouds.domain.Credentials.Builder;
84  import org.jclouds.io.Payload;
85  import org.jclouds.logging.Logger;
86  import org.jclouds.predicates.RetryablePredicate;
87  import org.jclouds.scriptbuilder.domain.Statement;
88  import org.jclouds.scriptbuilder.domain.Statements;
89  import org.jclouds.scriptbuilder.functions.InitAdminAccess;
90  import org.jclouds.util.Maps2;
91  import org.jclouds.util.Strings2;
92  
93  import com.google.common.base.Function;
94  import com.google.common.base.Predicate;
95  import com.google.common.base.Supplier;
96  import com.google.common.base.Throwables;
97  import com.google.common.collect.ImmutableMap;
98  import com.google.common.collect.ImmutableSet;
99  import com.google.common.collect.Iterables;
100 import com.google.common.collect.LinkedHashMultimap;
101 import com.google.common.collect.Multimap;
102 
103 /**
104  * 
105  * @author Adrian Cole
106  */
107 @Singleton
108 public class BaseComputeService implements ComputeService {
109 
110    @Resource
111    @Named(ComputeServiceConstants.COMPUTE_LOGGER)
112    protected Logger logger = Logger.NULL;
113 
114    protected final ComputeServiceContext context;
115    protected final Map<String, Credentials> credentialStore;
116 
117    private final Supplier<Set<? extends Image>> images;
118    private final Supplier<Set<? extends Hardware>> hardwareProfiles;
119    private final Supplier<Set<? extends Location>> locations;
120    private final ListNodesStrategy listNodesStrategy;
121    private final GetNodeMetadataStrategy getNodeMetadataStrategy;
122    private final CreateNodesInGroupThenAddToSet runNodesAndAddToSetStrategy;
123    private final RebootNodeStrategy rebootNodeStrategy;
124    private final DestroyNodeStrategy destroyNodeStrategy;
125    private final ResumeNodeStrategy resumeNodeStrategy;
126    private final SuspendNodeStrategy suspendNodeStrategy;
127    private final Provider<TemplateBuilder> templateBuilderProvider;
128    private final Provider<TemplateOptions> templateOptionsProvider;
129    private final Predicate<NodeMetadata> nodeRunning;
130    private final Predicate<NodeMetadata> nodeTerminated;
131    private final Predicate<NodeMetadata> nodeSuspended;
132    private final InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory;
133    private final Timeouts timeouts;
134    private final InitAdminAccess initAdminAccess;
135    private final PersistNodeCredentials persistNodeCredentials;
136    private final RunScriptOnNode.Factory runScriptOnNodeFactory;
137    private final ExecutorService executor;
138 
139    @Inject
140    protected BaseComputeService(ComputeServiceContext context, Map<String, Credentials> credentialStore,
141             @Memoized Supplier<Set<? extends Image>> images,
142             @Memoized Supplier<Set<? extends Hardware>> hardwareProfiles,
143             @Memoized Supplier<Set<? extends Location>> locations, ListNodesStrategy listNodesStrategy,
144             GetNodeMetadataStrategy getNodeMetadataStrategy,
145             CreateNodesInGroupThenAddToSet runNodesAndAddToSetStrategy, RebootNodeStrategy rebootNodeStrategy,
146             DestroyNodeStrategy destroyNodeStrategy, ResumeNodeStrategy resumeNodeStrategy,
147             SuspendNodeStrategy suspendNodeStrategy, Provider<TemplateBuilder> templateBuilderProvider,
148             Provider<TemplateOptions> templateOptionsProvider,
149             @Named("NODE_RUNNING") Predicate<NodeMetadata> nodeRunning,
150             @Named("NODE_TERMINATED") Predicate<NodeMetadata> nodeTerminated,
151             @Named("NODE_SUSPENDED") Predicate<NodeMetadata> nodeSuspended,
152             InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory, InitAdminAccess initAdminAccess,
153             RunScriptOnNode.Factory runScriptOnNodeFactory, PersistNodeCredentials persistNodeCredentials,
154             Timeouts timeouts, @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor) {
155       this.context = checkNotNull(context, "context");
156       this.credentialStore = checkNotNull(credentialStore, "credentialStore");
157       this.images = checkNotNull(images, "images");
158       this.hardwareProfiles = checkNotNull(hardwareProfiles, "hardwareProfiles");
159       this.locations = checkNotNull(locations, "locations");
160       this.listNodesStrategy = checkNotNull(listNodesStrategy, "listNodesStrategy");
161       this.getNodeMetadataStrategy = checkNotNull(getNodeMetadataStrategy, "getNodeMetadataStrategy");
162       this.runNodesAndAddToSetStrategy = checkNotNull(runNodesAndAddToSetStrategy, "runNodesAndAddToSetStrategy");
163       this.rebootNodeStrategy = checkNotNull(rebootNodeStrategy, "rebootNodeStrategy");
164       this.resumeNodeStrategy = checkNotNull(resumeNodeStrategy, "resumeNodeStrategy");
165       this.suspendNodeStrategy = checkNotNull(suspendNodeStrategy, "suspendNodeStrategy");
166       this.destroyNodeStrategy = checkNotNull(destroyNodeStrategy, "destroyNodeStrategy");
167       this.templateBuilderProvider = checkNotNull(templateBuilderProvider, "templateBuilderProvider");
168       this.templateOptionsProvider = checkNotNull(templateOptionsProvider, "templateOptionsProvider");
169       this.nodeRunning = checkNotNull(nodeRunning, "nodeRunning");
170       this.nodeTerminated = checkNotNull(nodeTerminated, "nodeTerminated");
171       this.nodeSuspended = checkNotNull(nodeSuspended, "nodeSuspended");
172       this.initScriptRunnerFactory = checkNotNull(initScriptRunnerFactory, "initScriptRunnerFactory");
173       this.timeouts = checkNotNull(timeouts, "timeouts");
174       this.initAdminAccess = checkNotNull(initAdminAccess, "initAdminAccess");
175       this.runScriptOnNodeFactory = checkNotNull(runScriptOnNodeFactory, "runScriptOnNodeFactory");
176       this.persistNodeCredentials = checkNotNull(persistNodeCredentials, "persistNodeCredentials");
177       this.executor = checkNotNull(executor, "executor");
178    }
179 
180    /**
181     * {@inheritDoc}
182     */
183    @Override
184    public ComputeServiceContext getContext() {
185       return context;
186    }
187 
188    /**
189     * {@inheritDoc}
190     */
191    @Override
192    public Set<? extends NodeMetadata> runNodesWithTag(String group, int count, Template template)
193             throws RunNodesException {
194       return createNodesInGroup(group, count, template);
195    }
196 
197    /**
198     * {@inheritDoc}
199     */
200    @Override
201    public Set<? extends NodeMetadata> runNodesWithTag(String group, int count, TemplateOptions templateOptions)
202             throws RunNodesException {
203       return createNodesInGroup(group, count, templateBuilder().any().options(templateOptions).build());
204    }
205 
206    /**
207     * {@inheritDoc}
208     */
209    @Override
210    public Set<? extends NodeMetadata> runNodesWithTag(String group, int count) throws RunNodesException {
211       return createNodesInGroup(group, count, templateOptions());
212    }
213 
214    @Override
215    public Set<? extends NodeMetadata> createNodesInGroup(String group, int count, Template template)
216             throws RunNodesException {
217       checkNotNull(group, "group cannot be null");
218       checkNotNull(template.getLocation(), "location");
219       logger.debug(">> running %d node%s group(%s) location(%s) image(%s) hardwareProfile(%s) options(%s)", count,
220                count > 1 ? "s" : "", group, template.getLocation().getId(), template.getImage().getId(), template
221                         .getHardware().getId(), template.getOptions());
222       Set<NodeMetadata> goodNodes = newLinkedHashSet();
223       Map<NodeMetadata, Exception> badNodes = newLinkedHashMap();
224       Multimap<NodeMetadata, CustomizationResponse> customizationResponses = LinkedHashMultimap.create();
225 
226       if (template.getOptions().getRunScript() != null)
227          initAdminAccess.visit(template.getOptions().getRunScript());
228 
229       Map<?, Future<Void>> responses = runNodesAndAddToSetStrategy.execute(group, count, template, goodNodes, badNodes,
230                customizationResponses);
231       Map<?, Exception> executionExceptions = awaitCompletion(responses, executor, null, logger, "runNodesWithTag("
232                + group + ")");
233       Function<NodeMetadata, NodeMetadata> fn = persistNodeCredentials.always(template.getOptions().getRunScript());
234       badNodes = Maps2.transformKeys(badNodes, fn);
235       goodNodes = ImmutableSet.copyOf(Iterables.transform(goodNodes, fn));
236       if (executionExceptions.size() > 0 || badNodes.size() > 0) {
237          throw new RunNodesException(group, count, template, goodNodes, executionExceptions, badNodes);
238       }
239       return goodNodes;
240    }
241 
242    @Override
243    public Set<? extends NodeMetadata> createNodesInGroup(String group, int count, TemplateOptions templateOptions)
244             throws RunNodesException {
245       return createNodesInGroup(group, count, templateBuilder().any().options(templateOptions).build());
246    }
247 
248    @Override
249    public Set<? extends NodeMetadata> createNodesInGroup(String group, int count) throws RunNodesException {
250       return createNodesInGroup(group, count, templateOptions());
251    }
252 
253    /**
254     * {@inheritDoc}
255     */
256    @Override
257    public void destroyNode(final String id) {
258       checkNotNull(id, "id");
259       logger.debug(">> destroying node(%s)", id);
260       final AtomicReference<NodeMetadata> node = new AtomicReference<NodeMetadata>();
261       RetryablePredicate<String> tester = new RetryablePredicate<String>(new Predicate<String>() {
262 
263          @Override
264          public boolean apply(String input) {
265             try {
266                NodeMetadata md = destroyNodeStrategy.destroyNode(id);
267                if (md != null)
268                   node.set(md);
269                return true;
270             } catch (IllegalStateException e) {
271                logger.warn("<< illegal state destroying node(%s)", id);
272                return false;
273             }
274          }
275 
276       }, timeouts.nodeRunning, 1000, TimeUnit.MILLISECONDS);
277       boolean successful = tester.apply(id) && (node.get() == null || nodeTerminated.apply(node.get()));
278       if (successful)
279          credentialStore.remove("node#" + id);
280       logger.debug("<< destroyed node(%s) success(%s)", id, successful);
281    }
282 
283    /**
284     * {@inheritDoc}
285     */
286    @Override
287    public Set<? extends NodeMetadata> destroyNodesMatching(Predicate<NodeMetadata> filter) {
288       logger.debug(">> destroying nodes matching(%s)", filter);
289       Set<NodeMetadata> set = newLinkedHashSet(transformParallel(nodesMatchingFilterAndNotTerminated(filter),
290                new Function<NodeMetadata, Future<NodeMetadata>>() {
291 
292                   // TODO make an async interface instead of re-wrapping
293                   @Override
294                   public Future<NodeMetadata> apply(final NodeMetadata from) {
295                      return executor.submit(new Callable<NodeMetadata>() {
296 
297                         @Override
298                         public NodeMetadata call() throws Exception {
299                            destroyNode(from.getId());
300                            return from;
301                         }
302 
303                         @Override
304                         public String toString() {
305                            return "destroyNode(" + from.getId() + ")";
306                         }
307                      });
308                   }
309 
310                }, executor, null, logger, "destroyNodesMatching(" + filter + ")"));
311       logger.debug("<< destroyed(%d)", set.size());
312       return set;
313    }
314 
315    Iterable<? extends NodeMetadata> nodesMatchingFilterAndNotTerminated(Predicate<NodeMetadata> filter) {
316       return filter(detailsOnAllNodes(), and(checkNotNull(filter, "filter"), not(TERMINATED)));
317    }
318 
319    /**
320     * @throws NoSuchElementException
321     *            if none found
322     */
323    Iterable<? extends NodeMetadata> nodesMatchingFilterAndNotTerminatedExceptionIfNotFound(
324             Predicate<NodeMetadata> filter) {
325       Iterable<? extends NodeMetadata> nodes = nodesMatchingFilterAndNotTerminated(filter);
326       if (Iterables.size(nodes) == 0)
327          throw new NoSuchElementException("no nodes matched filter: " + filter);
328       return nodes;
329    }
330 
331    /**
332     * {@inheritDoc}
333     */
334    @Override
335    public Set<ComputeMetadata> listNodes() {
336       logger.debug(">> listing nodes");
337       Set<ComputeMetadata> set = newLinkedHashSet(listNodesStrategy.listNodes());
338       logger.debug("<< list(%d)", set.size());
339       return set;
340    }
341 
342    /**
343     * {@inheritDoc}
344     */
345    @Override
346    public Set<? extends NodeMetadata> listNodesDetailsMatching(Predicate<ComputeMetadata> filter) {
347       checkNotNull(filter, "filter");
348       logger.debug(">> listing node details matching(%s)", filter);
349       Set<NodeMetadata> set = newLinkedHashSet(listNodesStrategy.listDetailsOnNodesMatching(filter));
350       logger.debug("<< list(%d)", set.size());
351       return set;
352    }
353 
354    /**
355     * {@inheritDoc}
356     */
357    @Override
358    public Set<? extends Hardware> listHardwareProfiles() {
359       return hardwareProfiles.get();
360    }
361 
362    /**
363     * {@inheritDoc}
364     */
365    @Override
366    public Set<? extends Image> listImages() {
367       return images.get();
368    }
369 
370    /**
371     * {@inheritDoc}
372     */
373    @Override
374    public Set<? extends Location> listAssignableLocations() {
375       return locations.get();
376    }
377 
378    /**
379     * {@inheritDoc}
380     */
381    @Override
382    public TemplateBuilder templateBuilder() {
383       return templateBuilderProvider.get();
384    }
385 
386    /**
387     * {@inheritDoc}
388     */
389    @Override
390    public NodeMetadata getNodeMetadata(String id) {
391       checkNotNull(id, "id");
392       return getNodeMetadataStrategy.getNode(id);
393    }
394 
395    /**
396     * {@inheritDoc}
397     */
398    @Override
399    public void rebootNode(String id) {
400       checkNotNull(id, "id");
401       logger.debug(">> rebooting node(%s)", id);
402       NodeMetadata node = rebootNodeStrategy.rebootNode(id);
403       boolean successful = nodeRunning.apply(node);
404       logger.debug("<< rebooted node(%s) success(%s)", id, successful);
405    }
406 
407    /**
408     * {@inheritDoc}
409     */
410    @Override
411    public void rebootNodesMatching(Predicate<NodeMetadata> filter) {
412       logger.debug(">> rebooting nodes matching(%s)", filter);
413       transformParallel(nodesMatchingFilterAndNotTerminatedExceptionIfNotFound(filter),
414                new Function<NodeMetadata, Future<Void>>() {
415                   // TODO use native async
416                   @Override
417                   public Future<Void> apply(NodeMetadata from) {
418                      rebootNode(from.getId());
419                      return immediateFuture(null);
420                   }
421 
422                }, executor, null, logger, "rebootNodesMatching(" + filter + ")");
423       logger.debug("<< rebooted");
424    }
425 
426    /**
427     * {@inheritDoc}
428     */
429    @Override
430    public void resumeNode(String id) {
431       checkNotNull(id, "id");
432       logger.debug(">> resuming node(%s)", id);
433       NodeMetadata node = resumeNodeStrategy.resumeNode(id);
434       boolean successful = nodeRunning.apply(node);
435       logger.debug("<< resumed node(%s) success(%s)", id, successful);
436    }
437 
438    /**
439     * {@inheritDoc}
440     */
441    @Override
442    public void resumeNodesMatching(Predicate<NodeMetadata> filter) {
443       logger.debug(">> resuming nodes matching(%s)", filter);
444       transformParallel(nodesMatchingFilterAndNotTerminatedExceptionIfNotFound(filter),
445                new Function<NodeMetadata, Future<Void>>() {
446                   // TODO use native async
447                   @Override
448                   public Future<Void> apply(NodeMetadata from) {
449                      resumeNode(from.getId());
450                      return immediateFuture(null);
451                   }
452 
453                }, executor, null, logger, "resumeNodesMatching(" + filter + ")");
454       logger.debug("<< resumed");
455    }
456 
457    /**
458     * {@inheritDoc}
459     */
460    @Override
461    public void suspendNode(String id) {
462       checkNotNull(id, "id");
463       logger.debug(">> suspending node(%s)", id);
464       NodeMetadata node = suspendNodeStrategy.suspendNode(id);
465       boolean successful = nodeSuspended.apply(node);
466       logger.debug("<< suspended node(%s) success(%s)", id, successful);
467    }
468 
469    /**
470     * {@inheritDoc}
471     */
472    @Override
473    public void suspendNodesMatching(Predicate<NodeMetadata> filter) {
474       logger.debug(">> suspending nodes matching(%s)", filter);
475       transformParallel(nodesMatchingFilterAndNotTerminatedExceptionIfNotFound(filter),
476                new Function<NodeMetadata, Future<Void>>() {
477                   // TODO use native async
478                   @Override
479                   public Future<Void> apply(NodeMetadata from) {
480                      suspendNode(from.getId());
481                      return immediateFuture(null);
482                   }
483 
484                }, executor, null, logger, "suspendNodesMatching(" + filter + ")");
485       logger.debug("<< suspended");
486    }
487 
488    /**
489     * {@inheritDoc}
490     */
491    @Override
492    public Map<NodeMetadata, ExecResponse> runScriptOnNodesMatching(Predicate<NodeMetadata> filter, Payload runScript)
493             throws RunScriptOnNodesException {
494       return runScriptOnNodesMatching(filter, runScript, RunScriptOptions.NONE);
495    }
496 
497    /**
498     * {@inheritDoc}
499     */
500    @Override
501    public Map<NodeMetadata, ExecResponse> runScriptOnNodesMatching(Predicate<NodeMetadata> filter, Payload runScript,
502             RunScriptOptions options) throws RunScriptOnNodesException {
503       try {
504          return runScriptOnNodesMatching(filter, Statements.exec(Strings2.toStringAndClose(checkNotNull(runScript,
505                   "runScript").getInput())), options);
506       } catch (IOException e) {
507          Throwables.propagate(e);
508          return null;
509       }
510    }
511 
512    /**
513     * {@inheritDoc}
514     */
515    @Override
516    public Map<NodeMetadata, ExecResponse> runScriptOnNodesMatching(Predicate<NodeMetadata> filter, String runScript)
517             throws RunScriptOnNodesException {
518       return runScriptOnNodesMatching(filter, Statements.exec(checkNotNull(runScript, "runScript")));
519    }
520 
521    /**
522     * {@inheritDoc}
523     */
524    @Override
525    public Map<NodeMetadata, ExecResponse> runScriptOnNodesMatching(Predicate<NodeMetadata> filter, Statement runScript)
526             throws RunScriptOnNodesException {
527       return runScriptOnNodesMatching(filter, runScript, RunScriptOptions.NONE);
528    }
529 
530    @Override
531    public Map<? extends NodeMetadata, ExecResponse> runScriptOnNodesMatching(Predicate<NodeMetadata> filter,
532             String runScript, RunScriptOptions options) throws RunScriptOnNodesException {
533       return runScriptOnNodesMatching(filter, Statements.exec(checkNotNull(runScript, "runScript")), options);
534    }
535 
536    /**
537     * {@inheritDoc}
538     */
539    @Override
540    public Map<NodeMetadata, ExecResponse> runScriptOnNodesMatching(Predicate<NodeMetadata> filter, Statement runScript,
541             RunScriptOptions options) throws RunScriptOnNodesException {
542 
543       checkNotNull(filter, "filter");
544       checkNotNull(runScript, "runScript");
545       checkNotNull(options, "options");
546 
547       Map<NodeMetadata, ExecResponse> goodNodes = newLinkedHashMap();
548       Map<NodeMetadata, Exception> badNodes = newLinkedHashMap();
549       Map<NodeMetadata, Future<ExecResponse>> responses = newLinkedHashMap();
550       Map<?, Exception> exceptions = ImmutableMap.<Object, Exception> of();
551 
552       initAdminAccess.visit(runScript);
553 
554       Iterable<? extends RunScriptOnNode> scriptRunners = transformNodesIntoInitializedScriptRunners(
555                nodesMatchingFilterAndNotTerminatedExceptionIfNotFound(filter), runScript, options, badNodes);
556       if (Iterables.size(scriptRunners) > 0) {
557          for (RunScriptOnNode runner : scriptRunners) {
558             responses.put(runner.getNode(), executor.submit(new RunScriptOnNodeAndAddToGoodMapOrPutExceptionIntoBadMap(
559                      runner, goodNodes, badNodes)));
560          }
561          exceptions = awaitCompletion(responses, executor, null, logger, "runScriptOnNodesMatching(" + filter + ")");
562       }
563 
564       Function<NodeMetadata, NodeMetadata> fn = persistNodeCredentials.ifAdminAccess(runScript);
565       badNodes = Maps2.transformKeys(badNodes, fn);
566       goodNodes = Maps2.transformKeys(goodNodes, fn);
567 
568       if (exceptions.size() > 0 || badNodes.size() > 0) {
569          throw new RunScriptOnNodesException(runScript, options, goodNodes, exceptions, badNodes);
570       }
571       return goodNodes;
572    }
573    
574    /**
575     * {@inheritDoc}
576     */
577    @Override
578    public ExecResponse runScriptOnNode(String id, String runScript) {
579       return runScriptOnNode(id, runScript, RunScriptOptions.NONE);
580    }
581    
582    /**
583     * {@inheritDoc}
584     */
585    @Override
586    public ExecResponse runScriptOnNode(String id, String runScript, RunScriptOptions options) {
587       return runScriptOnNode(id, Statements.exec(checkNotNull(runScript, "runScript")), options);
588    }
589    
590    /**
591     * {@inheritDoc}
592     */
593    @Override
594    public ExecResponse runScriptOnNode(String id, Statement runScript) {
595       return runScriptOnNode(id, runScript, RunScriptOptions.NONE);
596    }
597    
598    /**
599     * {@inheritDoc}
600     */
601    @Override
602    public ExecResponse runScriptOnNode(String id, Statement runScript, RunScriptOptions options) {
603       NodeMetadata node = this.getNodeMetadata(id);
604       if (node == null)
605          throw new NoSuchElementException(id);
606       if (node.getState() != NodeState.RUNNING)
607          throw new IllegalStateException("node " + id
608                   + " needs to be running before executing a script on it. current state: " + node.getState());
609       initAdminAccess.visit(runScript);
610       node = updateNodeWithCredentialsIfPresent(node, options);
611       ExecResponse response = runScriptOnNodeFactory.create(node, runScript, options).init().call();
612       persistNodeCredentials.ifAdminAccess(runScript).apply(node);
613       return response;
614    }
615 
616    private Iterable<? extends RunScriptOnNode> transformNodesIntoInitializedScriptRunners(
617             Iterable<? extends NodeMetadata> nodes, Statement script, RunScriptOptions options,
618             Map<NodeMetadata, Exception> badNodes) {
619       return filter(transformParallel(nodes, new TransformNodesIntoInitializedScriptRunners(script, options, badNodes),
620                executor, null, logger, "initialize script runners"), notNull());
621    }
622 
623    private Set<? extends NodeMetadata> detailsOnAllNodes() {
624       return newLinkedHashSet(listNodesStrategy.listDetailsOnNodesMatching(all()));
625    }
626 
627    @Override
628    public TemplateOptions templateOptions() {
629       return templateOptionsProvider.get();
630    }
631 
632    protected NodeMetadata updateNodeWithCredentialsIfPresent(NodeMetadata node, RunScriptOptions options) {
633       checkNotNull(node, "node");
634       if (options.getOverridingCredentials() != null) {
635          Builder<? extends Credentials> builder = node.getCredentials() != null ? node.getCredentials().toBuilder()
636                   : new Credentials.Builder<Credentials>();
637          if (options.getOverridingCredentials().identity != null)
638             builder.identity(options.getOverridingCredentials().identity);
639          if (options.getOverridingCredentials().credential != null)
640             builder.credential(options.getOverridingCredentials().credential);
641          node = NodeMetadataBuilder.fromNodeMetadata(node).credentials(builder.build()).build();
642       }
643       return node;
644    }
645    
646    private final class TransformNodesIntoInitializedScriptRunners implements
647             Function<NodeMetadata, Future<RunScriptOnNode>> {
648       private final Map<NodeMetadata, Exception> badNodes;
649       private final Statement script;
650       private final RunScriptOptions options;
651 
652       private TransformNodesIntoInitializedScriptRunners(Statement script, RunScriptOptions options,
653                Map<NodeMetadata, Exception> badNodes) {
654          this.badNodes = checkNotNull(badNodes, "badNodes");
655          this.script = checkNotNull(script, "script");
656          this.options = checkNotNull(options, "options");
657       }
658 
659       @Override
660       public Future<RunScriptOnNode> apply(NodeMetadata node) {
661          node = updateNodeWithCredentialsIfPresent(node, options);
662          return executor.submit(initScriptRunnerFactory.create(node, script, options, badNodes));
663       }
664 
665    }
666 
667 }