ghost_mode.h

Go to the documentation of this file.

#ifndef FPMAS_GHOST_MODE_H
#define FPMAS_GHOST_MODE_H
 
#include "fpmas/communication/communication.h"
#include "fpmas/api/graph/distributed_graph.h"
#include "fpmas/utils/macros.h"
#include "../data_update_pack.h"
#include "../synchro.h"
#include "single_thread_mutex.h"
 
namespace fpmas { namespace synchro {
    namespace ghost {
 
        using api::graph::LocationState;
        template<typename T>
            class GhostMutex : public SingleThreadMutex<T> {
                public:
                    using SingleThreadMutex<T>::SingleThreadMutex;
 
                    const T& read() override {return this->data();};
                    void releaseRead() override {};
                    T& acquire() override {return this->data();};
                    void releaseAcquire() override {};
                    void synchronize() override {};
            };
 
        template<typename T>
            class GhostDataSync : public api::synchro::DataSync<T> {
                public:
                    typedef api::communication::TypedMpi<NodeUpdatePack<T>> DataMpi;
                    typedef api::communication::TypedMpi<DistributedId> IdMpi;
 
                private:
                    DataMpi& data_mpi;
                    IdMpi& id_mpi;
 
                    api::graph::DistributedGraph<T>& graph;
                    std::unordered_map<int, std::vector<DistributedId>> buildRequests();
                    std::unordered_map<int, std::vector<DistributedId>> buildRequests(
                            std::unordered_set<api::graph::DistributedNode<T>*> nodes
                            );
                    void _synchronize(
                            std::unordered_map<int, std::vector<DistributedId>> requests
                            );
 
                public:
                    GhostDataSync(
                            DataMpi& data_mpi, IdMpi& id_mpi,
                            api::graph::DistributedGraph<T>& graph
                            )
                        : data_mpi(data_mpi), id_mpi(id_mpi), graph(graph) {}
 
                    void synchronize() override;
 
                    void synchronize(
                            std::unordered_set<api::graph::DistributedNode<T>*> nodes
                            ) override;
            };
 
        template<typename T>
            void GhostDataSync<T>::_synchronize(
                    std::unordered_map<int, std::vector<DistributedId>> requests) {
                FPMAS_LOGI(
                        graph.getMpiCommunicator().getRank(), "GHOST_MODE",
                        "Synchronizing graph data...", ""
                        );
                requests = id_mpi.migrate(std::move(requests));
 
                std::unordered_map<int, std::vector<NodeUpdatePack<T>>> updated_data;
                updated_data.reserve(graph.getMpiCommunicator().getSize());
                for(auto list : requests) {
                    updated_data[list.first].reserve(list.second.size());
                    for(auto id : list.second) {
                        FPMAS_LOGV(
                                graph.getMpiCommunicator().getRank(), "GHOST_MODE",
                                "Export %s to %i", FPMAS_C_STR(id), list.first
                                );
                        auto node = graph.getNode(id);
                        updated_data[list.first].emplace_back(
                                id, node->data(), node->getWeight()
                                );
                    }
                }
 
                // TODO : Should use data update packs.
                updated_data = data_mpi.migrate(std::move(updated_data));
                for(auto list : updated_data) {
                    for(auto& data : list.second) {
                        auto local_node = graph.getNode(data.id);
                        synchro::DataUpdate<T>::update(
                                local_node->data(),
                                std::move(data.updated_data)
                                );
                        local_node->setWeight(data.updated_weight);
                    }
                }
 
                FPMAS_LOGI(
                        graph.getMpiCommunicator().getRank(), "GHOST_MODE",
                        "Graph data synchronized.", ""
                        );
            }
 
        template<typename T>
             std::unordered_map<int, std::vector<DistributedId>> GhostDataSync<T>
             ::buildRequests(std::unordered_set<api::graph::DistributedNode<T>*> nodes) {
                 std::unordered_map<int, std::vector<DistributedId>> requests;
                 requests.reserve(graph.getMpiCommunicator().getSize());
                 for(auto node : nodes) {
                     if(node->state() == fpmas::api::graph::DISTANT) {
                         FPMAS_LOGV(
                                 graph.getMpiCommunicator().getRank(), "GHOST_MODE",
                                 "Request %s from %i",
                                 FPMAS_C_STR(node->getId()), node->location());
 
                         requests[node->location()].emplace_back(node->getId());
                     }
                 }
                 return requests;
             }
 
        template<typename T>
            void GhostDataSync<T>::synchronize(
                    std::unordered_set<api::graph::DistributedNode<T>*> nodes) {
                FPMAS_LOGI(
                        graph.getMpiCommunicator().getRank(), "GHOST_MODE",
                        "Synchronizing graph data...", ""
                        );
 
                _synchronize(buildRequests(nodes));
                for(auto node : nodes)
                    node->mutex()->synchronize();
            }
 
        template<typename T>
             std::unordered_map<int, std::vector<DistributedId>> GhostDataSync<T>
			 ::buildRequests() {
                 std::unordered_map<int, std::vector<DistributedId>> requests;
                 requests.reserve(graph.getMpiCommunicator().getSize());
                 for(auto node : graph.getLocationManager().getDistantNodes()) {
                     FPMAS_LOGV(
                             graph.getMpiCommunicator().getRank(), "GHOST_MODE",
                             "Request %s from %i",
                             FPMAS_C_STR(node.first), node.second->location()
                             );
                     requests[node.second->location()].emplace_back(node.first);
                 }
                 return requests;
             }
 
        template<typename T>
            void GhostDataSync<T>::synchronize() {
                FPMAS_LOGI(
                        graph.getMpiCommunicator().getRank(), "GHOST_MODE",
                        "Synchronizing graph data...", "");
 
                _synchronize(buildRequests());
                for(auto node : graph.getNodes())
                    node.second->mutex()->synchronize();
            }
 
 
        template<typename T>
            class GhostSyncLinkerBase : public api::synchro::SyncLinker<T> {
                public:
                    typedef api::graph::DistributedEdge<T> EdgeApi;
                    typedef api::graph::DistributedNode<T> NodeApi;
                    typedef api::utils::PtrWrapper<EdgeApi> EdgePtr;
                    typedef api::communication::TypedMpi<EdgePtr> EdgeMpi;
                    typedef api::communication::TypedMpi<DistributedId> IdMpi;
                private:
                    std::vector<EdgePtr> link_buffer;
                    std::unordered_map<int, std::vector<DistributedId>> unlink_migration;
                    std::unordered_map<int, std::vector<DistributedId>> remove_node_buffer;
                    std::vector<NodeApi*> local_nodes_to_remove;
 
                    EdgeMpi& edge_mpi;
                    IdMpi& id_mpi;
                    api::graph::DistributedGraph<T>& graph;
 
                protected:
                    void synchronize_links();
 
                public:
                    GhostSyncLinkerBase(
                            EdgeMpi& edge_mpi, IdMpi& id_mpi,
                            api::graph::DistributedGraph<T>& graph)
                        : edge_mpi(edge_mpi), id_mpi(id_mpi), graph(graph) {}
 
                    void link(EdgeApi* edge) override;
 
                    void unlink(EdgeApi* edge) override;
 
                    void removeNode(NodeApi* node) override;
            };
 
        template<typename T>
            void GhostSyncLinkerBase<T>::synchronize_links() {
                FPMAS_LOGI(graph.getMpiCommunicator().getRank(),
                        "GHOST_MODE", "Synchronizing graph links...", "");
                /*
                 * Migrate links
                 */
                std::vector<EdgePtr> edges_to_clear;
                std::unordered_map<int, std::vector<EdgePtr>> link_migration;
                for(auto edge : link_buffer) {
                    auto src = edge->getSourceNode();
                    if(src->state() == LocationState::DISTANT) {
                        link_migration[src->location()].push_back(edge);
                    }
                    auto tgt = edge->getTargetNode();
                    if(tgt->state() == LocationState::DISTANT) {
                        link_migration[tgt->location()].push_back(edge);
                    }
                    if(src->state() == LocationState::DISTANT
                            && tgt->state() == LocationState::DISTANT) {
                        edges_to_clear.push_back(edge);
                    }
                }
                link_migration = edge_mpi.migrate(std::move(link_migration));
 
                for(auto import_list : link_migration) {
                    for (auto edge : import_list.second) {
                        graph.importEdge(edge);
                    }
                }
                link_buffer.clear();
 
                /*
                 * Migrate node removal
                 */
                remove_node_buffer = id_mpi.migrate(std::move(remove_node_buffer));
                for(auto import_list : remove_node_buffer) {
                    for(DistributedId node_id : import_list.second) {
                        auto* node = graph.getNode(node_id);
                        for(auto edge : node->getOutgoingEdges())
                            graph.unlink(edge);
                        for(auto edge : node->getIncomingEdges())
                            graph.unlink(edge);
                    }
                }
 
                /*
                 * Migrate unlinks
                 */
                unlink_migration = id_mpi.migrate(std::move(unlink_migration));
                for(auto import_list : unlink_migration) {
                    for(DistributedId id : import_list.second) {
                        const auto& edges = graph.getEdges();
                        auto it = edges.find(id);
                        if(it != edges.end())
                            graph.erase(it->second);
                    }
                }
                unlink_migration.clear();
 
                for(auto edge : edges_to_clear)
                    graph.erase(edge);
 
                for(auto import_list : remove_node_buffer)
                    for(auto node_id : import_list.second)
                        local_nodes_to_remove.push_back(graph.getNode(node_id));
                remove_node_buffer.clear();
 
                for(auto node : local_nodes_to_remove) {
                    graph.erase(node);
                }
                local_nodes_to_remove.clear();
 
                FPMAS_LOGI(graph.getMpiCommunicator().getRank(),
                        "GHOST_MODE", "Graph links synchronized.", "");
            }
 
        template<typename T>
            void GhostSyncLinkerBase<T>::link(EdgeApi * edge) {
                if(edge->state() == LocationState::DISTANT) {
                    link_buffer.push_back(const_cast<EdgeApi*>(edge));
                }
            }
 
        template<typename T>
            void GhostSyncLinkerBase<T>::unlink(EdgeApi * edge) {
                if(edge->state() == LocationState::DISTANT) {
                    link_buffer.erase(
                            std::remove(link_buffer.begin(), link_buffer.end(), edge),
                            link_buffer.end()
                            );
                    auto src = edge->getSourceNode();
                    if(src->state() == LocationState::DISTANT) {
                        unlink_migration[src->location()].push_back(edge->getId());
                    }
                    auto tgt = edge->getTargetNode();
                    if(tgt->state() == LocationState::DISTANT) {
                        unlink_migration[tgt->location()].push_back(edge->getId());
                    }
                }
            }
 
        template<typename T>
            void GhostSyncLinkerBase<T>::removeNode(NodeApi* node) {
                if(node->state() == LocationState::DISTANT) {
                    remove_node_buffer[node->location()].push_back(node->getId());
                } else {
                    for(auto edge : node->getOutgoingEdges())
                        graph.unlink(edge);
                    for(auto edge : node->getIncomingEdges())
                        graph.unlink(edge);
                }
                local_nodes_to_remove.push_back(node);
            }
 
        template<typename T>
            class GhostSyncLinker : public GhostSyncLinkerBase<T> {
                public:
                    using GhostSyncLinkerBase<T>::GhostSyncLinkerBase;
 
                    void synchronize() override;
            };
 
        template<typename T>
            void GhostSyncLinker<T>::synchronize() {
                this->synchronize_links();
            }
 
        template<typename T, template<typename> class Mutex>
            class GhostMode : public api::synchro::SyncMode<T> {
                communication::TypedMpi<DistributedId> id_mpi;
                communication::TypedMpi<NodeUpdatePack<T>> data_mpi;
                communication::TypedMpi<api::utils::PtrWrapper<api::graph::DistributedEdge<T>>> edge_mpi;
 
                GhostDataSync<T> data_sync;
                GhostSyncLinker<T> sync_linker;
 
                public:
                GhostMode(
                        api::graph::DistributedGraph<T>& graph,
                        api::communication::MpiCommunicator& comm)
                    : id_mpi(comm), data_mpi(comm), edge_mpi(comm),
                    data_sync(data_mpi, id_mpi, graph), sync_linker(edge_mpi, id_mpi, graph) {}
 
                Mutex<T>* buildMutex(api::graph::DistributedNode<T>* node) override {
                    return new Mutex<T>(node->data());
                };
 
                GhostDataSync<T>& getDataSync() override {return data_sync;}
 
                GhostSyncLinker<T>& getSyncLinker() override {return sync_linker;}
            };
    }
 
    template<typename T>
        using GhostMode = ghost::GhostMode<T, ghost::GhostMutex>;
}}
#endif