diff --git a/src/main/java/com/sparrowwallet/sparrow/control/PayNymCell.java b/src/main/java/com/sparrowwallet/sparrow/control/PayNymCell.java
index a68f7278..de1f1329 100644
--- a/src/main/java/com/sparrowwallet/sparrow/control/PayNymCell.java
+++ b/src/main/java/com/sparrowwallet/sparrow/control/PayNymCell.java
@@ -17,14 +17,16 @@ import java.util.Optional;
 
 public class PayNymCell extends ListCell<PayNym> {
     private final PayNymController payNymController;
+    private final boolean contact;
 
-    public PayNymCell(PayNymController payNymController) {
+    public PayNymCell(PayNymController payNymController, boolean contact) {
         super();
         setAlignment(Pos.CENTER_LEFT);
         setContentDisplay(ContentDisplay.LEFT);
         getStyleClass().add("paynym-cell");
         setPrefHeight(50);
         this.payNymController = payNymController;
+        this.contact = contact;
     }
 
     @Override
@@ -52,7 +54,7 @@ public class PayNymCell extends ListCell<PayNym> {
             labelBox.getChildren().add(label);
             pane.setLeft(labelBox);
 
-            if(getListView().getUserData() == Boolean.TRUE) {
+            if(getListView().getUserData() == Boolean.TRUE || (!contact && payNymController != null && payNymController.isFollowing(payNym) == Boolean.FALSE)) {
                 HBox hBox = new HBox();
                 hBox.setAlignment(Pos.CENTER);
                 Button button = new Button("Add Contact");
@@ -62,7 +64,7 @@ public class PayNymCell extends ListCell<PayNym> {
                     button.setDisable(true);
                     payNymController.followPayNym(payNym.paymentCode());
                 });
-            } else if(payNymController != null) {
+            } else if(contact && payNymController != null) {
                 HBox hBox = new HBox();
                 hBox.setAlignment(Pos.CENTER);
                 pane.setRight(hBox);
@@ -89,7 +91,7 @@ public class PayNymCell extends ListCell<PayNym> {
             setText(null);
             setGraphic(pane);
 
-            if(payNymController != null && payNymController.isLinked(payNym)) {
+            if(contact && payNymController != null && payNymController.isLinked(payNym)) {
                 setContextMenu(new PayNymCellContextMenu(payNym));
             } else {
                 setContextMenu(null);
diff --git a/src/main/java/com/sparrowwallet/sparrow/paynym/PayNymController.java b/src/main/java/com/sparrowwallet/sparrow/paynym/PayNymController.java
index 2285d69f..4cfb5b99 100644
--- a/src/main/java/com/sparrowwallet/sparrow/paynym/PayNymController.java
+++ b/src/main/java/com/sparrowwallet/sparrow/paynym/PayNymController.java
@@ -151,7 +151,7 @@ public class PayNymController {
         findPayNym.setVisible(false);
 
         followingList.setCellFactory(param -> {
-            return new PayNymCell(this);
+            return new PayNymCell(this, true);
         });
 
         followingList.getSelectionModel().selectedItemProperty().addListener((observable, oldValue, payNym) -> {
@@ -159,7 +159,7 @@ public class PayNymController {
         });
 
         followersList.setCellFactory(param -> {
-            return new PayNymCell(null);
+            return new PayNymCell(this, false);
         });
 
         followersList.setSelectionModel(new NoSelectionModel<>());
@@ -282,30 +282,43 @@ public class PayNymController {
     public void followPayNym(PaymentCode contact) {
         PayNymService payNymService = AppServices.getPayNymService();
         Wallet masterWallet = getMasterWallet();
+        retrievePayNymProgress.setVisible(true);
         payNymService.getAuthToken(masterWallet, new HashMap<>()).subscribe(authToken -> {
             String signature = payNymService.getSignature(masterWallet, authToken);
             payNymService.followPaymentCode(contact, authToken, signature).subscribe(followMap -> {
                 refresh();
             }, error -> {
+                retrievePayNymProgress.setVisible(false);
                 log.error("Could not follow payment code", error);
                 Optional<ButtonType> optResponse = showErrorDialog("Error retrieving PayNym", "Could not follow payment code. Try again?", ButtonType.CANCEL, ButtonType.OK);
                 if(optResponse.isPresent() && optResponse.get().equals(ButtonType.OK)) {
                     followPayNym(contact);
                 } else {
                     followingList.refresh();
+                    followersList.refresh();
                 }
             });
         }, error -> {
+            retrievePayNymProgress.setVisible(false);
             log.error("Could not follow payment code", error);
             Optional<ButtonType> optResponse = showErrorDialog("Error retrieving PayNym", "Could not follow payment code. Try again?", ButtonType.CANCEL, ButtonType.OK);
             if(optResponse.isPresent() && optResponse.get().equals(ButtonType.OK)) {
                 followPayNym(contact);
             } else {
                 followingList.refresh();
+                followersList.refresh();
             }
         });
     }
 
+    public Boolean isFollowing(PayNym payNym) {
+        if(followingList.getItems() != null) {
+            return followingList.getItems().stream().anyMatch(following -> payNym.paymentCode().equals(following.paymentCode()));
+        }
+
+        return null;
+    }
+
     public boolean isLinked(PayNym payNym) {
         PaymentCode externalPaymentCode = payNym.paymentCode();
         return getMasterWallet().getChildWallet(externalPaymentCode, payNym.segwit() ? ScriptType.P2WPKH : ScriptType.P2PKH) != null;